Some of my performance tests for tabled hit truncation again:
[zaitcev@hitlain tests]$ ./poke5 -v -h niphredil.zaitcev.lan -u auser
-p apass -b test -o -k testkey-hitlain/73b84a11e6d83c65e45853338d646042
-f testdir/73b84a11e6d83c65e45853338d646042
* About to connect() to niphredil.zaitcev.lan port 80 (#0)
* Trying fec0::1:219:b9ff:fe58:7ad6... * TCP_NODELAY set
* connected
* Connected to niphredil.zaitcev.lan (fec0::1:219:b9ff:fe59:7ad6) port 80 (#0)
> PUT /test/testkey-hitlain/73b84a11e6d83c65e45853338d HTTP/1.1
Accept: */*
Host: niphredil.zaitcev.lan
Date: Tue, 20 Jul 2010 01:07:33 +0000
Authorization: AWS testuser:RefcbVYgr2m9KTRxOrCfr4zzfPE=
Content-Length: 214745088
Expect: 100-continue
* The requested URL returned error: 403
As you can see, the path in PUT is truncated, and this causes 403
since it's included into a hash.
The patch addresses this issue and a bunch of other fixed-size
strings before we hit that.
Signed-off-by: Pete Zaitcev <zaitcev@xxxxxxxxxx>
---
lib/hstor.c | 147 +++++++++++++++++++++++++++++++++++---------------
1 file changed, 104 insertions(+), 43 deletions(-)
diff --git a/lib/hstor.c b/lib/hstor.c
index 5dfff27..6c67bfa 100644
--- a/lib/hstor.c
+++ b/lib/hstor.c
@@ -179,7 +179,8 @@ next:
struct hstor_blist *hstor_list_buckets(struct hstor_client *hstor)
{
struct http_req req;
- char datestr[80], timestr[64], hmac[64], auth[128], host[80], url[80];
+ char datestr[80], timestr[64], hmac[64], auth[128];
+ char *host, *url;
struct curl_slist *headers = NULL;
struct hstor_blist *blist;
xmlDocPtr doc;
@@ -190,7 +191,7 @@ struct hstor_blist *hstor_list_buckets(struct hstor_client *hstor)
all_data = g_byte_array_new();
if (!all_data)
- return NULL;
+ goto err_data;
memset(&req, 0, sizeof(req));
req.method = "GET";
@@ -204,14 +205,15 @@ struct hstor_blist *hstor_list_buckets(struct hstor_client *hstor)
hreq_sign(&req, NULL, hstor->key, hmac);
sprintf(auth, "Authorization: AWS %s:%s", hstor->user, hmac);
- sprintf(host, "Host: %s", hstor->host);
+ if (asprintf(&host, "Host: %s", hstor->host) < 0)
+ goto err_host;
+ if (asprintf(&url, "http://%s/";, hstor->acc) < 0)
+ goto err_url;
headers = curl_slist_append(headers, host);
headers = curl_slist_append(headers, datestr);
headers = curl_slist_append(headers, auth);
- snprintf(url, sizeof(url), "http://%s/";, hstor->acc);
-
curl_easy_reset(hstor->curl);
if (hstor->verbose)
curl_easy_setopt(hstor->curl, CURLOPT_VERBOSE, 1);
@@ -289,15 +291,20 @@ struct hstor_blist *hstor_list_buckets(struct hstor_client *hstor)
xmlFreeDoc(doc);
g_byte_array_free(all_data, TRUE);
- all_data = NULL;
+ free(url);
+ free(host);
return blist;
err_out_doc:
xmlFreeDoc(doc);
err_out:
+ free(url);
+err_url:
+ free(host);
+err_host:
g_byte_array_free(all_data, TRUE);
- all_data = NULL;
+err_data:
return NULL;
}
@@ -305,12 +312,13 @@ static bool __hstor_ad_bucket(struct hstor_client *hstor, const char *name,
bool delete)
{
struct http_req req;
- char datestr[80], timestr[64], hmac[64], auth[128], host[80],
- url[80], orig_path[80];
+ char datestr[80], timestr[64], hmac[64], auth[128];
+ char *host, *url, *orig_path;
struct curl_slist *headers = NULL;
int rc;
- sprintf(orig_path, "/%s/", name);
+ if (asprintf(&orig_path, "/%s/", name) < 0)
+ goto err_spath;
memset(&req, 0, sizeof(req));
req.method = delete ? "DELETE" : "PUT";
@@ -324,8 +332,10 @@ static bool __hstor_ad_bucket(struct hstor_client *hstor, const char *name,
hreq_sign(&req, NULL, hstor->key, hmac);
sprintf(auth, "Authorization: AWS %s:%s", hstor->user, hmac);
- sprintf(host, "Host: %s", hstor->host);
- snprintf(url, sizeof(url), "http://%s/%s/";, hstor->acc, name);
+ if (asprintf(&host, "Host: %s", hstor->host) < 0)
+ goto err_host;
+ if (asprintf(&url, "http://%s/%s/";, hstor->acc, name) < 0)
+ goto err_url;
headers = curl_slist_append(headers, host);
headers = curl_slist_append(headers, datestr);
@@ -344,7 +354,17 @@ static bool __hstor_ad_bucket(struct hstor_client *hstor, const char *name,
curl_slist_free_all(headers);
+ free(url);
+ free(host);
+ free(orig_path);
return (rc == 0);
+
+err_url:
+ free(host);
+err_host:
+ free(orig_path);
+err_spath:
+ return false;
}
bool hstor_add_bucket(struct hstor_client *hstor, const char *name)
@@ -362,15 +382,15 @@ bool hstor_get(struct hstor_client *hstor, const char *bucket, const char *key,
void *user_data, bool want_headers)
{
struct http_req req;
- char datestr[80], timestr[64], hmac[64], auth[128], host[80],
- url[80], *orig_path, *stmp;
+ char datestr[80], timestr[64], hmac[64], auth[128];
+ char *host, *url, *orig_path;
struct curl_slist *headers = NULL;
int rc;
- if (asprintf(&stmp, "/%s/%s", bucket, key) < 0)
- return false;
+ if (asprintf(&orig_path, "/%s/%s", bucket, key) < 0)
+ goto err_spath;
- orig_path = huri_field_escape(stmp, PATH_ESCAPE_MASK);
+ orig_path = huri_field_escape(orig_path, PATH_ESCAPE_MASK);
memset(&req, 0, sizeof(req));
req.method = "GET";
@@ -384,8 +404,10 @@ bool hstor_get(struct hstor_client *hstor, const char *bucket, const char *key,
hreq_sign(&req, NULL, hstor->key, hmac);
sprintf(auth, "Authorization: AWS %s:%s", hstor->user, hmac);
- sprintf(host, "Host: %s", hstor->host);
- snprintf(url, sizeof(url), "http://%s%s";, hstor->acc, orig_path);
+ if (asprintf(&host, "Host: %s", hstor->host) < 0)
+ goto err_host;
+ if (asprintf(&url, "http://%s%s";, hstor->acc, orig_path) < 0)
+ goto err_url;
headers = curl_slist_append(headers, host);
headers = curl_slist_append(headers, datestr);
@@ -406,9 +428,18 @@ bool hstor_get(struct hstor_client *hstor, const char *bucket, const char *key,
rc = curl_easy_perform(hstor->curl);
curl_slist_free_all(headers);
+ free(url);
+ free(host);
free(orig_path);
return (rc == 0);
+
+err_url:
+ free(host);
+err_host:
+ free(orig_path);
+err_spath:
+ return false;
}
void *hstor_get_inline(struct hstor_client *hstor, const char *bucket, const char *key,
@@ -442,15 +473,16 @@ bool hstor_put(struct hstor_client *hstor, const char *bucket, const char *key,
uint64_t len, void *user_data, char **user_hdrs)
{
struct http_req req;
- char datestr[80], timestr[64], hmac[64], auth[128], host[80],
- url[80], *orig_path, *stmp, *uhdr_buf = NULL;
+ char datestr[80], timestr[64], hmac[64], auth[128];
+ char *host, *url, *orig_path;
+ char *uhdr_buf = NULL;
struct curl_slist *headers = NULL;
int rc = -1;
- if (asprintf(&stmp, "/%s/%s", bucket, key) < 0)
- return false;
+ if (asprintf(&orig_path, "/%s/%s", bucket, key) < 0)
+ goto err_spath;
- orig_path = huri_field_escape(stmp, PATH_ESCAPE_MASK);
+ orig_path = huri_field_escape(orig_path, PATH_ESCAPE_MASK);
memset(&req, 0, sizeof(req));
req.method = "PUT";
@@ -477,7 +509,7 @@ bool hstor_put(struct hstor_client *hstor, const char *bucket, const char *key,
/* alloc buf to hold all hdr strings */
uhdr_buf = calloc(1, uhdr_len);
if (!uhdr_buf)
- goto out;
+ goto err_ubuf;
/* copy and nul-terminate hdr keys and values for signing */
idx = 0;
@@ -509,8 +541,10 @@ bool hstor_put(struct hstor_client *hstor, const char *bucket, const char *key,
hreq_sign(&req, NULL, hstor->key, hmac);
sprintf(auth, "Authorization: AWS %s:%s", hstor->user, hmac);
- sprintf(host, "Host: %s", hstor->host);
- snprintf(url, sizeof(url), "http://%s%s";, hstor->acc, orig_path);
+ if (asprintf(&host, "Host: %s", hstor->host) < 0)
+ goto err_host;
+ if (asprintf(&url, "http://%s%s";, hstor->acc, orig_path) < 0)
+ goto err_url;
headers = curl_slist_append(headers, host);
headers = curl_slist_append(headers, datestr);
@@ -533,11 +567,20 @@ bool hstor_put(struct hstor_client *hstor, const char *bucket, const char *key,
rc = curl_easy_perform(hstor->curl);
curl_slist_free_all(headers);
+ free(url);
+ free(host);
free(orig_path);
-
-out:
free(uhdr_buf);
return (rc == 0);
+
+err_url:
+ free(host);
+err_host:
+ free(uhdr_buf);
+err_ubuf:
+ free(orig_path);
+err_spath:
+ return false;
}
struct hstor_put_info {
@@ -572,15 +615,15 @@ bool hstor_put_inline(struct hstor_client *hstor, const char *bucket, const char
bool hstor_del(struct hstor_client *hstor, const char *bucket, const char *key)
{
struct http_req req;
- char datestr[80], timestr[64], hmac[64], auth[128], host[80],
- url[80], *orig_path, *stmp;
+ char datestr[80], timestr[64], hmac[64], auth[128];
+ char *host, *url, *orig_path;
struct curl_slist *headers = NULL;
int rc;
- if (asprintf(&stmp, "/%s/%s", bucket, key) < 0)
- return false;
+ if (asprintf(&orig_path, "/%s/%s", bucket, key) < 0)
+ goto err_spath;
- orig_path = huri_field_escape(stmp, PATH_ESCAPE_MASK);
+ orig_path = huri_field_escape(orig_path, PATH_ESCAPE_MASK);
memset(&req, 0, sizeof(req));
req.method = "DELETE";
@@ -594,8 +637,10 @@ bool hstor_del(struct hstor_client *hstor, const char *bucket, const char *key)
hreq_sign(&req, NULL, hstor->key, hmac);
sprintf(auth, "Authorization: AWS %s:%s", hstor->user, hmac);
- sprintf(host, "Host: %s", hstor->host);
- snprintf(url, sizeof(url), "http://%s%s";, hstor->acc, orig_path);
+ if (asprintf(&host, "Host: %s", hstor->host) < 0)
+ goto err_host;
+ if (asprintf(&url, "http://%s%s";, hstor->acc, orig_path) < 0)
+ goto err_url;
headers = curl_slist_append(headers, host);
headers = curl_slist_append(headers, datestr);
@@ -613,9 +658,18 @@ bool hstor_del(struct hstor_client *hstor, const char *bucket, const char *key)
rc = curl_easy_perform(hstor->curl);
curl_slist_free_all(headers);
+ free(url);
+ free(host);
free(orig_path);
return (rc == 0);
+
+err_url:
+ free(host);
+err_host:
+ free(orig_path);
+err_spath:
+ return false;
}
static GString *append_qparam(GString *str, const char *key, const char *val,
@@ -761,8 +815,8 @@ struct hstor_keylist *hstor_keys(struct hstor_client *hstor, const char *bucket,
const char *delim, unsigned int max_keys)
{
struct http_req req;
- char datestr[80], timestr[64], hmac[64], auth[128], host[80];
- char orig_path[strlen(bucket) + 8];
+ char datestr[80], timestr[64], hmac[64], auth[128];
+ char *host, *orig_path;
struct curl_slist *headers = NULL;
struct hstor_keylist *keylist;
xmlDocPtr doc;
@@ -775,9 +829,10 @@ struct hstor_keylist *hstor_keys(struct hstor_client *hstor, const char *bucket,
all_data = g_byte_array_new();
if (!all_data)
- return NULL;
+ goto err_data;
- sprintf(orig_path, "/%s/", bucket);
+ if (asprintf(&orig_path, "/%s/", bucket) < 0)
+ goto err_spath;
memset(&req, 0, sizeof(req));
req.method = "GET";
@@ -791,7 +846,8 @@ struct hstor_keylist *hstor_keys(struct hstor_client *hstor, const char *bucket,
hreq_sign(&req, NULL, hstor->key, hmac);
sprintf(auth, "Authorization: AWS %s:%s", hstor->user, hmac);
- sprintf(host, "Host: %s", hstor->host);
+ if (asprintf(&host, "Host: %s", hstor->host) < 0)
+ goto err_host;
headers = curl_slist_append(headers, host);
headers = curl_slist_append(headers, datestr);
@@ -928,16 +984,21 @@ struct hstor_keylist *hstor_keys(struct hstor_client *hstor, const char *bucket,
}
xmlFreeDoc(doc);
+ free(host);
+ free(orig_path);
g_byte_array_free(all_data, TRUE);
- all_data = NULL;
return keylist;
err_out_doc:
xmlFreeDoc(doc);
err_out:
+ free(host);
+err_host:
+ free(orig_path);
+err_spath:
g_byte_array_free(all_data, TRUE);
- all_data = NULL;
+err_data:
return NULL;
}
--
To unsubscribe from this list: send the line "unsubscribe hail-devel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
