diff options
author | Yehuda Sadeh <yehuda@inktank.com> | 2012-11-08 17:17:43 -0800 |
---|---|---|
committer | Yehuda Sadeh <yehuda@inktank.com> | 2012-11-08 17:17:43 -0800 |
commit | e5124ced11e4150bae1b4b9b3fcc30b1deebbcac (patch) | |
tree | 00ffe3fb767d594f5b5a1bb334543c0376e57037 | |
parent | b2d4c491c7e75da76230b569d90a479739cdf4da (diff) | |
parent | bfc49049e36c006123295fb038361ae1b63f6ede (diff) | |
download | ceph-e5124ced11e4150bae1b4b9b3fcc30b1deebbcac.tar.gz |
Merge branch 'wip-post-cleaned' into wip-rgw-integration
Conflicts:
src/rgw/rgw_common.cc
src/rgw/rgw_op.cc
Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>
-rw-r--r-- | src/Makefile.am | 12 | ||||
-rw-r--r-- | src/json_spirit/json_spirit_reader_template.h | 4 | ||||
-rw-r--r-- | src/rgw/rgw_common.cc | 55 | ||||
-rw-r--r-- | src/rgw/rgw_common.h | 10 | ||||
-rw-r--r-- | src/rgw/rgw_html_errors.h | 2 | ||||
-rw-r--r-- | src/rgw/rgw_json.cc | 255 | ||||
-rw-r--r-- | src/rgw/rgw_json.h | 92 | ||||
-rw-r--r-- | src/rgw/rgw_jsonparser.cc | 80 | ||||
-rw-r--r-- | src/rgw/rgw_op.cc | 120 | ||||
-rw-r--r-- | src/rgw/rgw_op.h | 54 | ||||
-rw-r--r-- | src/rgw/rgw_policy_s3.cc | 290 | ||||
-rw-r--r-- | src/rgw/rgw_policy_s3.h | 56 | ||||
-rw-r--r-- | src/rgw/rgw_rest.cc | 92 | ||||
-rw-r--r-- | src/rgw/rgw_rest.h | 17 | ||||
-rw-r--r-- | src/rgw/rgw_rest_s3.cc | 777 | ||||
-rw-r--r-- | src/rgw/rgw_rest_s3.h | 48 | ||||
-rw-r--r-- | src/rgw/rgw_string.h | 94 |
17 files changed, 2020 insertions, 38 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index e1875256d27..51abc35fa46 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -151,7 +151,7 @@ base: ceph-mon ceph-osd ceph-mds \ ceph cephfs \ ceph-syn \ rados radosgw librados-config \ - ceph-conf monmaptool osdmaptool crushtool ceph-authtool \ + ceph-conf monmaptool osdmaptool crushtool jsontest ceph-authtool \ init-ceph mkcephfs @@ -333,6 +333,7 @@ librgw_a_SOURCES = \ rgw/rgw_fcgi.cc \ rgw/rgw_xml.cc \ rgw/rgw_usage.cc \ + rgw/rgw_json.cc \ rgw/rgw_user.cc \ rgw/rgw_tools.cc \ rgw/rgw_rados.cc \ @@ -342,6 +343,7 @@ librgw_a_SOURCES = \ rgw/rgw_formats.cc \ rgw/rgw_log.cc \ rgw/rgw_multi.cc \ + rgw/rgw_policy_s3.cc \ rgw/rgw_gc.cc \ rgw/rgw_multi_del.cc \ rgw/rgw_env.cc @@ -377,6 +379,11 @@ rgw_multiparser_CXXFLAGS = ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS} rgw_multiparser_LDADD = $(my_radosgw_ldadd) bin_DEBUGPROGRAMS += rgw_multiparser +rgw_jsonparser_SOURCES = rgw/rgw_jsonparser.cc +rgw_jsonparser_CXXFLAGS = ${CRYPTO_CXXFLAGS} ${AM_CXXFLAGS} +rgw_jsonparser_LDADD = $(my_radosgw_ldadd) +bin_DEBUGPROGRAMS += rgw_jsonparser + endif # librbd @@ -1810,12 +1817,15 @@ noinst_HEADERS = \ rgw/rgw_client_io.h\ rgw/rgw_fcgi.h\ rgw/rgw_xml.h\ + rgw/rgw_json.h\ rgw/rgw_cache.h\ rgw/rgw_common.h\ + rgw/rgw_string.h\ rgw/rgw_formats.h\ rgw/rgw_html_errors.h\ rgw/rgw_log.h\ rgw/rgw_multi.h\ + rgw/rgw_policy_s3.h\ rgw/rgw_gc.h\ rgw/rgw_multi_del.h\ rgw/rgw_op.h\ diff --git a/src/json_spirit/json_spirit_reader_template.h b/src/json_spirit/json_spirit_reader_template.h index 93f64e49dc0..b814e916255 100644 --- a/src/json_spirit/json_spirit_reader_template.h +++ b/src/json_spirit/json_spirit_reader_template.h @@ -468,7 +468,7 @@ namespace json_spirit ;
members_
- = pair_ >> *( ',' >> pair_ )
+ = pair_ >> *( ',' >> pair_ | ch_p(',') )
;
pair_
@@ -484,7 +484,7 @@ namespace json_spirit ;
elements_
- = value_ >> *( ',' >> value_ )
+ = value_ >> *( ',' >> value_ | ch_p(',') )
;
string_
diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index e76554d2f31..c030e3466cd 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -2,6 +2,7 @@ #include "rgw_common.h" #include "rgw_acl.h" +#include "rgw_string.h" #include "common/ceph_crypto.h" #include "common/armor.h" @@ -81,7 +82,7 @@ is_clear() const bool rgw_err:: is_err() const { - return !(http_ret >= 200 && http_ret <= 299); + return !(http_ret >= 200 && http_ret <= 399); } @@ -158,6 +159,17 @@ string rgw_string_unquote(const string& s) return s.substr(1, len - 2); } +static void trim_whitespace(const string& src, string& dst) +{ + const char *spacestr = " \t\n\r\f\v"; + int start = src.find_first_not_of(spacestr); + if (start < 0) + return; + + int end = src.find_last_not_of(spacestr); + dst = src.substr(start, end - start + 1); +} + static bool check_str_end(const char *s) { if (!s) @@ -226,6 +238,34 @@ bool parse_rfc2616(const char *s, struct tm *t) return parse_rfc850(s, t) || parse_asctime(s, t) || parse_rfc1123(s, t) || parse_rfc1123_alt(s,t); } +bool parse_iso8601(const char *s, struct tm *t) +{ + memset(t, 0, sizeof(*t)); + const char *p = strptime(s, "%Y-%m-%dT%T", t); + if (!p) { + dout(0) << "parse_iso8601 failed" << dendl; + return false; + } + string str; + trim_whitespace(p, str); + if (str.size() == 1 && str[0] == 'Z') + return true; + + if (str.size() != 5) { + return false; + } + if (str[0] != '.' || + str[str.size() - 1] != 'Z') + return false; + + uint32_t ms; + int r = stringtoul(str.substr(1, 3), &ms); + if (r < 0) + return false; + + return true; +} + int parse_time(const char *time_str, time_t *time) { struct tm tm; @@ -238,7 +278,7 @@ int parse_time(const char *time_str, time_t *time) return 0; } -int parse_date(string& date, uint64_t *epoch, string *out_date, string *out_time) +int parse_date(const string& date, uint64_t *epoch, string *out_date, string *out_time) { struct tm tm; @@ -575,17 +615,6 @@ int RGWUserCaps::parse_cap_perm(const string& str, uint32_t *perm) return 0; } -static void trim_whitespace(const string& src, string& dst) -{ - const char *spacestr = " \t\n\r\f\v"; - int start = src.find_first_not_of(spacestr); - if (start < 0) - return; - - int end = src.find_last_not_of(spacestr); - dst = src.substr(start, end - start + 1); -} - int RGWUserCaps::get_cap(const string& cap, string& type, uint32_t *pperm) { int pos = cap.find('='); diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index ddd71f45884..0ba9e95c80b 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -39,10 +39,12 @@ using ceph::crypto::MD5; #define RGW_ATTR_PREFIX "user.rgw." +#define RGW_AMZ_META_PREFIX "x-amz-meta-" + #define RGW_ATTR_ACL RGW_ATTR_PREFIX "acl" #define RGW_ATTR_ETAG RGW_ATTR_PREFIX "etag" #define RGW_ATTR_BUCKETS RGW_ATTR_PREFIX "buckets" -#define RGW_ATTR_META_PREFIX RGW_ATTR_PREFIX "x-amz-meta-" +#define RGW_ATTR_META_PREFIX RGW_ATTR_PREFIX RGW_AMZ_META_PREFIX #define RGW_ATTR_CONTENT_TYPE RGW_ATTR_PREFIX "content_type" #define RGW_ATTR_CACHE_CONTROL RGW_ATTR_PREFIX "cache_control" #define RGW_ATTR_CONTENT_DISP RGW_ATTR_PREFIX "content_disposition" @@ -79,6 +81,7 @@ using ceph::crypto::MD5; #define STATUS_ACCEPTED 1901 #define STATUS_NO_CONTENT 1902 #define STATUS_PARTIAL_CONTENT 1903 +#define STATUS_REDIRECT 1904 #define ERR_INVALID_BUCKET_NAME 2000 #define ERR_INVALID_OBJECT_NAME 2001 @@ -102,6 +105,7 @@ using ceph::crypto::MD5; #define ERR_TOO_LARGE 2019 #define ERR_TOO_MANY_BUCKETS 2020 #define ERR_INVALID_REQUEST 2021 +#define ERR_TOO_SMALL 2022 #define ERR_USER_SUSPENDED 2100 #define ERR_INTERNAL_ERROR 2200 @@ -565,6 +569,7 @@ struct req_state { ceph::Formatter *formatter; string decoded_uri; string request_uri; + string script_uri; string request_params; const char *host; const char *method; @@ -993,7 +998,8 @@ extern string rgw_string_unquote(const string& s); /** time parsing */ extern int parse_time(const char *time_str, time_t *time); extern bool parse_rfc2616(const char *s, struct tm *t); -extern int parse_date(string& date, uint64_t *epoch, string *out_date = NULL, string *out_time = NULL); +extern bool parse_iso8601(const char *s, struct tm *t); +extern int parse_date(const string& date, uint64_t *epoch, string *out_date = NULL, string *out_time = NULL); /** Check if the req_state's user has the necessary permissions * to do the requested action */ diff --git a/src/rgw/rgw_html_errors.h b/src/rgw/rgw_html_errors.h index c4335bcaf0a..254af46e663 100644 --- a/src/rgw/rgw_html_errors.h +++ b/src/rgw/rgw_html_errors.h @@ -15,6 +15,7 @@ const static struct rgw_html_errors RGW_HTML_ERRORS[] = { { STATUS_ACCEPTED, 202, "Accepted" }, { STATUS_NO_CONTENT, 204, "NoContent" }, { STATUS_PARTIAL_CONTENT, 206, "" }, + { STATUS_REDIRECT, 303, "" }, { ERR_NOT_MODIFIED, 304, "NotModified" }, { EINVAL, 400, "InvalidArgument" }, { ERR_INVALID_REQUEST, 400, "InvalidRequest" }, @@ -27,6 +28,7 @@ const static struct rgw_html_errors RGW_HTML_ERRORS[] = { { ERR_INVALID_PART_ORDER, 400, "InvalidPartOrder" }, { ERR_REQUEST_TIMEOUT, 400, "RequestTimeout" }, { ERR_TOO_LARGE, 400, "EntityTooLarge" }, + { ERR_TOO_SMALL, 400, "EntityTooSmall" }, { ERR_TOO_MANY_BUCKETS, 400, "TooManyBuckets" }, { ERR_LENGTH_REQUIRED, 411, "MissingContentLength" }, { EACCES, 403, "AccessDenied" }, diff --git a/src/rgw/rgw_json.cc b/src/rgw/rgw_json.cc new file mode 100644 index 00000000000..1a6fc32dea9 --- /dev/null +++ b/src/rgw/rgw_json.cc @@ -0,0 +1,255 @@ +#include <iostream> +#include <include/types.h> + +#include "rgw_json.h" + +// for testing DELETE ME +#include <fstream> + +using namespace std; +using namespace json_spirit; + +JSONObjIter::JSONObjIter() +{ +} + +JSONObjIter::~JSONObjIter() +{ +} + +void JSONObjIter::set(const JSONObjIter::map_iter_t &_cur, const JSONObjIter::map_iter_t &_last) +{ + cur = _cur; + last = _last; +} + +void JSONObjIter::operator++() +{ + if (cur != last) + ++cur; +}; + +JSONObj *JSONObjIter::operator*() +{ + return cur->second; +}; + +// does not work, FIXME +ostream& operator<<(ostream& out, JSONObj& obj) { + out << obj.name << ": " << obj.data_string; + return out; +} + +JSONObj::~JSONObj() +{ + multimap<string, JSONObj *>::iterator iter; + for (iter = children.begin(); iter != children.end(); ++iter) { + JSONObj *obj = iter->second; + delete obj; + } +} + + +void JSONObj::add_child(string el, JSONObj *obj) +{ + cout << "add_child: " << name << " <- " << el << endl; + children.insert(pair<string, JSONObj *>(el, obj)); +} + +bool JSONObj::get_attr(string name, string& attr) +{ + map<string, string>::iterator iter = attr_map.find(name); + if (iter == attr_map.end()) + return false; + attr = iter->second; + return true; +} + +JSONObjIter JSONObj::find(string name) +{ + JSONObjIter iter; + map<string, JSONObj *>::iterator first; + map<string, JSONObj *>::iterator last; + first = children.find(name); + last = children.upper_bound(name); + iter.set(first, last); + return iter; +} + +JSONObjIter JSONObj::find_first() +{ + JSONObjIter iter; + iter.set(children.begin(), children.end()); + cout << "count=" << children.size() << endl; + for (map<string, JSONObj *>:: iterator i = children.begin(); i != children.end(); ++i) { + cout << "child: " << i->first << endl; + } + return iter; +} + +JSONObjIter JSONObj::find_first(string name) +{ + JSONObjIter iter; + map<string, JSONObj *>::iterator first; + first = children.find(name); + iter.set(first, children.end()); + return iter; +} + +/* accepts a JSON Array or JSON Object contained in + * a JSON Spirit Value, v, and creates a JSONObj for each + * child contained in v + */ +void JSONObj::handle_value(Value v) +{ + if (v.type() == obj_type) { + Object temp_obj = v.get_obj(); + for (Object::size_type i = 0; i < temp_obj.size(); i++) { + Pair temp_pair = temp_obj[i]; + string temp_name = temp_pair.name_; + Value temp_value = temp_pair.value_; + JSONObj *child = new JSONObj; + child->init(this, temp_value, temp_name); + add_child(temp_name, child); + } + } else if (v.type() == array_type) { + Array temp_array = v.get_array(); + Value value; + + for (unsigned j = 0; j < temp_array.size(); j++) { + Value cur = temp_array[j]; + + if (cur.type() == obj_type) { + handle_value(cur); + } else { + string temp_name; + + JSONObj *child = new JSONObj; + child->init(this, cur, temp_name); + add_child(child->get_name(), child); + } + } + } +} + +void JSONObj::init(JSONObj *p, Value v, string n) +{ + name = n; + parent = p; + data = v; + + handle_value(v); + if (v.type() == str_type) + data_string = v.get_str(); + else + data_string = write(v, raw_utf8); + attr_map.insert(pair<string,string>(name, data_string)); +} + +JSONObj *JSONObj::get_parent() +{ + return parent; +} + +bool JSONObj::is_object() +{ + cout << data.type() << std::endl; + return (data.type() == obj_type); +} + +bool JSONObj::is_array() +{ + return (data.type() == array_type); +} + +vector<string> JSONObj::get_array_elements() +{ + vector<string> elements; + Array temp_array; + + if (data.type() == array_type) + temp_array = data.get_array(); + + int array_size = temp_array.size(); + if (array_size > 0) + for (int i = 0; i < array_size; i++) { + Value temp_value = temp_array[i]; + string temp_string; + temp_string = write(temp_value, raw_utf8); + elements.push_back(temp_string); + } + + return elements; +} + +RGWJSONParser::RGWJSONParser() : buf_len(0), success(true) +{ +} + +RGWJSONParser::~RGWJSONParser() +{ +} + + + +void RGWJSONParser::handle_data(const char *s, int len) +{ + json_buffer.append(s, len); // check for problems with null termination FIXME + buf_len += len; +} + +// parse a supplied JSON fragment +bool RGWJSONParser::parse(const char *buf_, int len) +{ + string json_string = buf_; + // make a substring to len + json_string = json_string.substr(0, len); + success = read(json_string, data); + if (success) + handle_value(data); + else + set_failure(); + + return success; +} + +// parse the internal json_buffer up to len +bool RGWJSONParser::parse(int len) +{ + string json_string = json_buffer.substr(0, len); + success = read(json_string, data); + if (success) + handle_value(data); + else + set_failure(); + + return success; +} + +// parse the complete internal json_buffer +bool RGWJSONParser::parse() +{ + success = read(json_buffer, data); + if (success) + handle_value(data); + else + set_failure(); + + return success; +} + +// parse a supplied ifstream, for testing. DELETE ME +bool RGWJSONParser::parse(const char *file_name) +{ + ifstream is(file_name); + success = read(is, data); + if (success) + handle_value(data); + else + set_failure(); + + return success; +} + + + diff --git a/src/rgw/rgw_json.h b/src/rgw/rgw_json.h new file mode 100644 index 00000000000..e11ab25b405 --- /dev/null +++ b/src/rgw/rgw_json.h @@ -0,0 +1,92 @@ +#ifndef RGW_JSON_H +#define RGW_JSON_H + +#include <iostream> +#include <include/types.h> + +// for testing DELETE ME +#include <fstream> + +#include "json_spirit/json_spirit.h" + + +using namespace std; +using namespace json_spirit; + + +class JSONObj; + +class JSONObjIter { + typedef map<string, JSONObj *>::iterator map_iter_t; + map_iter_t cur; + map_iter_t last; + +public: + JSONObjIter(); + ~JSONObjIter(); + void set(const JSONObjIter::map_iter_t &_cur, const JSONObjIter::map_iter_t &_end); + + void operator++(); + JSONObj *operator*(); + + bool end() { + return (cur == last); + } +}; + +class JSONObj +{ + JSONObj *parent; +protected: + string name; // corresponds to obj_type in XMLObj + Value data; + string data_string; + multimap<string, JSONObj *> children; + map<string, string> attr_map; + void handle_value(Value v); + +public: + + JSONObj() : parent(NULL){}; + + virtual ~JSONObj(); + + void init(JSONObj *p, Value v, string n); + + string& get_name() { return name; } + string& get_data() { return data_string; } + JSONObj *get_parent(); + void add_child(string el, JSONObj *child); + bool get_attr(string name, string& attr); + JSONObjIter find(string name); + JSONObjIter find_first(); + JSONObjIter find_first(string name); + + friend ostream& operator<<(ostream& out, JSONObj& obj); // does not work, FIXME + + bool is_array(); + bool is_object(); + vector<string> get_array_elements(); +}; + +class RGWJSONParser : public JSONObj +{ + int buf_len; + string json_buffer; + bool success; +public: + RGWJSONParser(); + virtual ~RGWJSONParser(); + void handle_data(const char *s, int len); + + bool parse(const char *buf_, int len); + bool parse(int len); + bool parse(); + bool parse(const char *file_name); + + const char *get_json() { return json_buffer.c_str(); } + void set_failure() { success = false; } +}; + + +#endif diff --git a/src/rgw/rgw_jsonparser.cc b/src/rgw/rgw_jsonparser.cc new file mode 100644 index 00000000000..7a1f11c325e --- /dev/null +++ b/src/rgw/rgw_jsonparser.cc @@ -0,0 +1,80 @@ +#include <string.h> + +#include <iostream> +#include <map> + +#include "include/types.h" + +#include "rgw_json.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +void dump_array(JSONObj *obj) +{ + + JSONObjIter iter = obj->find_first(); + + for (; !iter.end(); ++iter) { + JSONObj *o = *iter; + cout << "data=" << o->get_data() << endl; + } + +} + +int main(int argc, char **argv) { + RGWJSONParser parser; + + char buf[1024]; + + for (;;) { + int done; + int len; + + len = fread(buf, 1, sizeof(buf), stdin); + if (ferror(stdin)) { + cerr << "read error" << std::endl; + exit(-1); + } + done = feof(stdin); + + bool ret = parser.parse(buf, len); + if (!ret) + cerr << "parse error" << std::endl; + + if (done) + break; + } + + JSONObjIter iter = parser.find_first(); + + for (; !iter.end(); ++iter) { + JSONObj *obj = *iter; + cout << "is_object=" << obj->is_object() << endl; + cout << "is_array=" << obj->is_array() << endl; + cout << "name=" << obj->get_name() << endl; + cout << "data=" << obj->get_data() << endl; + } + + iter = parser.find_first("conditions"); + if (!iter.end()) { + JSONObj *obj = *iter; + + JSONObjIter iter2 = obj->find_first(); + for (; !iter2.end(); ++iter2) { + JSONObj *child = *iter2; + cout << "is_object=" << child->is_object() << endl; + cout << "is_array=" << child->is_array() << endl; + if (child->is_array()) { + dump_array(child); + } + cout << "name=" << child->get_name() << endl; + cout << "data=" << child->get_data() << endl; + } + } + + + exit(0); +} + diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index b4a50ceb8a8..a3a84aaea13 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -283,7 +283,7 @@ static int read_policy(RGWRados *store, struct req_state *s, RGWBucketInfo& buck * only_bucket: If true, reads the bucket ACL rather than the object ACL. * Returns: 0 on success, -ERR# otherwise. */ -static int build_policies(RGWRados *store, struct req_state *s, bool only_bucket, bool prefetch_data) +int rgw_build_policies(RGWRados *store, struct req_state *s, bool only_bucket, bool prefetch_data) { int ret = 0; string obj_str; @@ -1373,6 +1373,122 @@ done: (ceph_clock_now(s->cct) - s->time)); } +int RGWPostObj::verify_permission() +{ + return 0; +} + +RGWPutObjProcessor *RGWPostObj::select_processor() +{ + RGWPutObjProcessor *processor; + + uint64_t part_size = s->cct->_conf->rgw_obj_stripe_size; + + if (s->content_length <= RGW_MAX_CHUNK_SIZE) + processor = new RGWPutObjProcessor_Plain(); + else + processor = new RGWPutObjProcessor_Atomic(part_size); + + return processor; +} + +void RGWPostObj::dispose_processor(RGWPutObjProcessor *processor) +{ + delete processor; +} + +void RGWPostObj::execute() +{ + RGWPutObjProcessor *processor = NULL; + char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; + unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; + MD5 hash; + bufferlist bl, aclbl; + int len = 0; + + // read in the data from the POST form + ret = get_params(); + if (ret < 0) + goto done; + + ret = verify_params(); + if (ret < 0) + goto done; + + if (!verify_bucket_permission(s, RGW_PERM_WRITE)) { + ret = -EACCES; + goto done; + } + + processor = select_processor(); + + ret = processor->prepare(store, s); + if (ret < 0) + goto done; + + while (data_pending) { + bufferlist data; + len = get_data(data); + + if (len < 0) { + ret = len; + goto done; + } + + if (!len) + break; + + void *handle; + const unsigned char *data_ptr = (const unsigned char *)data.c_str(); + + ret = processor->handle_data(data, ofs, &handle); + if (ret < 0) + goto done; + + hash.Update(data_ptr, len); + + ret = processor->throttle_data(handle); + if (ret < 0) + goto done; + + ofs += len; + + if (ofs > max_len) { + ret = -ERR_TOO_LARGE; + goto done; + } + } + + if (len < min_len) { + ret = -ERR_TOO_SMALL; + goto done; + } + + s->obj_size = ofs; + + hash.Final(m); + buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, calc_md5); + + policy.encode(aclbl); + etag = calc_md5; + + bl.append(etag.c_str(), etag.size() + 1); + attrs[RGW_ATTR_ETAG] = bl; + attrs[RGW_ATTR_ACL] = aclbl; + + if (content_type.size()) { + bufferlist ct_bl; + ct_bl.append(content_type.c_str(), content_type.size() + 1); + attrs[RGW_ATTR_CONTENT_TYPE] = ct_bl; + } + + ret = processor->complete(etag, attrs); + +done: + dispose_processor(processor); +} + + int RGWPutMetadata::verify_permission() { if (!verify_object_permission(s, RGW_PERM_WRITE)) @@ -2161,7 +2277,7 @@ int RGWHandler::init(RGWRados *_store, struct req_state *_s, RGWClientIO *cio) int RGWHandler::do_read_permissions(RGWOp *op, bool only_bucket) { - int ret = build_policies(store, s, only_bucket, op->prefetch_data()); + int ret = rgw_build_policies(store, s, only_bucket, op->prefetch_data()); if (ret < 0) { ldout(s->cct, 10) << "read_permissions on " << s->bucket << ":" <<s->object_str << " only_bucket=" << only_bucket << " ret=" << ret << dendl; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 0c664890019..23f45a6448a 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -9,7 +9,10 @@ #ifndef CEPH_RGW_OP_H #define CEPH_RGW_OP_H +#include <limits.h> + #include <string> +#include <map> #include "rgw_common.h" #include "rgw_rados.h" @@ -22,6 +25,8 @@ struct req_state; class RGWHandler; void rgw_get_request_metadata(struct req_state *s, map<string, bufferlist>& attrs); +int rgw_build_policies(RGWRados *store, struct req_state *s, bool only_bucket, bool prefetch_data); + /** * Provide the base class for all ops. @@ -312,6 +317,55 @@ public: virtual const char *name() { return "put_obj"; } }; +class RGWPostObj : public RGWOp { + + friend class RGWPutObjProcessor; + +protected: + off_t min_len; + off_t max_len; + int ret; + int len; + off_t ofs; + const char *supplied_md5_b64; + const char *supplied_etag; + string etag; + string boundary; + bool data_pending; + string content_type; + RGWAccessControlPolicy policy; + map<string, bufferlist> attrs; + +public: + RGWPostObj() {} + + virtual void init(RGWRados *store, struct req_state *s, RGWHandler *h) { + RGWOp::init(store, s, h); + min_len = 0; + max_len = LLONG_MAX; + ret = 0; + len = 0; + ofs = 0; + supplied_md5_b64 = NULL; + supplied_etag = NULL; + etag = ""; + boundary = ""; + data_pending = false; + policy.set_ctx(s->cct); + } + + int verify_permission(); + void execute(); + + RGWPutObjProcessor *select_processor(); + void dispose_processor(RGWPutObjProcessor *processor); + + virtual int get_params() = 0; + virtual int get_data(bufferlist& bl) = 0; + virtual void send_response() = 0; + virtual const char *name() { return "post_obj"; } +}; + class RGWPutMetadata : public RGWOp { protected: int ret; diff --git a/src/rgw/rgw_policy_s3.cc b/src/rgw/rgw_policy_s3.cc new file mode 100644 index 00000000000..91adbe85233 --- /dev/null +++ b/src/rgw/rgw_policy_s3.cc @@ -0,0 +1,290 @@ + +#include <errno.h> + +#include "rgw_policy_s3.h" +#include "rgw_json.h" +#include "rgw_common.h" + + +#define dout_subsys ceph_subsys_rgw + +class RGWPolicyCondition { +protected: + string v1; + string v2; + + virtual bool check(const string& first, const string& second, string& err_msg) = 0; + +public: + virtual ~RGWPolicyCondition() {} + + void set_vals(const string& _v1, const string& _v2) { + v1 = _v1; + v2 = _v2; + } + + bool check(RGWPolicyEnv *env, map<string, bool, ltstr_nocase>& checked_vars, string& err_msg) { + string first, second; + env->get_value(v1, first, checked_vars); + env->get_value(v2, second, checked_vars); + + dout(1) << "policy condition check " << v1 << " [" << first << "] " << v2 << " [" << second << "]" << dendl; + bool ret = check(first, second, err_msg); + if (!ret) { + err_msg.append(": "); + err_msg.append(v1); + err_msg.append(", "); + err_msg.append(v2); + } + return ret; + } + +}; + + +class RGWPolicyCondition_StrEqual : public RGWPolicyCondition { +protected: + bool check(const string& first, const string& second, string& msg) { + bool ret = first.compare(second) == 0; + if (!ret) { + msg = "Policy condition failed: eq"; + } + return ret; + } +}; + +class RGWPolicyCondition_StrStartsWith : public RGWPolicyCondition { +protected: + bool check(const string& first, const string& second, string& msg) { + bool ret = first.compare(0, second.size(), second) == 0; + if (!ret) { + msg = "Policy condition failed: starts-with"; + } + return ret; + } +}; + +void RGWPolicyEnv::add_var(const string& name, const string& value) +{ + vars[name] = value; +} + +bool RGWPolicyEnv::get_var(const string& name, string& val) +{ + map<string, string, ltstr_nocase>::iterator iter = vars.find(name); + if (iter == vars.end()) + return false; + + val = iter->second; + + return true; +} + +bool RGWPolicyEnv::get_value(const string& s, string& val, map<string, bool, ltstr_nocase>& checked_vars) +{ + if (s.empty() || s[0] != '$') { + val = s; + return true; + } + + const string& var = s.substr(1); + checked_vars[var] = true; + + return get_var(var, val); +} + + +bool RGWPolicyEnv::match_policy_vars(map<string, bool, ltstr_nocase>& policy_vars, string& err_msg) +{ + map<string, string, ltstr_nocase>::iterator iter; + string ignore_prefix = "x-ignore-"; + for (iter = vars.begin(); iter != vars.end(); ++iter) { + const string& var = iter->first; + if (strncasecmp(ignore_prefix.c_str(), var.c_str(), ignore_prefix.size()) == 0) + continue; + if (policy_vars.count(var) == 0) { + err_msg = "Policy missing condition: "; + err_msg.append(iter->first); + dout(1) << "env var missing in policy: " << iter->first << dendl; + return false; + } + } + return true; +} + +RGWPolicy::~RGWPolicy() +{ + list<RGWPolicyCondition *>::iterator citer; + for (citer = conditions.begin(); citer != conditions.end(); ++citer) { + RGWPolicyCondition *cond = *citer; + delete cond; + } +} + +int RGWPolicy::set_expires(const string& e) +{ + struct tm t; + if (!parse_iso8601(e.c_str(), &t)) + return -EINVAL; + + expires = timegm(&t); + + return 0; +} + +int RGWPolicy::add_condition(const string& op, const string& first, const string& second, string& err_msg) +{ + RGWPolicyCondition *cond = NULL; + if (stringcasecmp(op, "eq") == 0) { + cond = new RGWPolicyCondition_StrEqual; + } else if (stringcasecmp(op, "starts-with") == 0) { + cond = new RGWPolicyCondition_StrStartsWith; + } else if (stringcasecmp(op, "content-length-range") == 0) { + off_t min, max; + int r = stringtoll(first, &min); + if (r < 0) { + err_msg = "Bad content-length-range param"; + dout(0) << "bad content-length-range param: " << first << dendl; + return r; + } + + r = stringtoll(second, &max); + if (r < 0) { + err_msg = "Bad content-length-range param"; + dout(0) << "bad content-length-range param: " << second << dendl; + return r; + } + + if (min > min_length) + min_length = min; + + if (max < max_length) + max_length = max; + + return 0; + } + + if (!cond) { + err_msg = "Invalid condition: "; + err_msg.append(op); + dout(0) << "invalid condition: " << op << dendl; + return -EINVAL; + } + + cond->set_vals(first, second); + + conditions.push_back(cond); + + return 0; +} + +int RGWPolicy::check(RGWPolicyEnv *env, string& err_msg) +{ + uint64_t now = ceph_clock_now(NULL).sec(); + if (expires <= now) { + dout(0) << "NOTICE: policy calculated as expired: " << expiration_str << dendl; + err_msg = "Policy expired"; + return -EACCES; // change to condition about expired policy following S3 + } + + list<pair<string, string> >::iterator viter; + for (viter = var_checks.begin(); viter != var_checks.end(); ++viter) { + pair<string, string>& p = *viter; + const string& name = p.first; + const string& check_val = p.second; + string val; + if (!env->get_var(name, val)) { + err_msg = "Policy check failed, variable not found: "; + err_msg.append(name); + return -EACCES; + } + + set_var_checked(name); + + dout(20) << "comparing " << name << " [" << val << "], " << check_val << dendl; + if (val.compare(check_val) != 0) { + err_msg = "Policy check failed, variable not met condition: "; + err_msg.append(name); + dout(1) << "policy check failed, val=" << val << " != " << check_val << dendl; + return -EACCES; + } + } + + list<RGWPolicyCondition *>::iterator citer; + for (citer = conditions.begin(); citer != conditions.end(); ++citer) { + RGWPolicyCondition *cond = *citer; + if (!cond->check(env, checked_vars, err_msg)) { + return -EACCES; + } + } + + if (!env->match_policy_vars(checked_vars, err_msg)) { + dout(1) << "missing policy condition" << dendl; + return -EACCES; + } + return 0; +} + + +int RGWPolicy::from_json(bufferlist& bl, string& err_msg) +{ + RGWJSONParser parser; + + if (!parser.parse(bl.c_str(), bl.length())) { + err_msg = "Malformed JSON"; + dout(0) << "malformed json" << dendl; + return -EINVAL; + } + + // as no time was included in the request, we hope that the user has included a short timeout + JSONObjIter iter = parser.find_first("expiration"); + if (iter.end()) { + err_msg = "Policy missing expiration"; + dout(0) << "expiration not found" << dendl; + return -EINVAL; // change to a "no expiration" error following S3 + } + + JSONObj *obj = *iter; + expiration_str = obj->get_data(); + int r = set_expires(expiration_str); + if (r < 0) { + err_msg = "Failed to parse policy expiration"; + return r; + } + + iter = parser.find_first("conditions"); + if (iter.end()) { + err_msg = "Policy missing conditions"; + dout(0) << "conditions not found" << dendl; + return -EINVAL; // change to a "no conditions" error following S3 + } + + obj = *iter; + + iter = obj->find_first(); + for (; !iter.end(); ++iter) { + JSONObj *child = *iter; + dout(20) << "is_object=" << child->is_object() << dendl; + dout(20) << "is_array=" << child->is_array() << dendl; + if (child->is_array()) { + JSONObjIter aiter = child->find_first(); + vector<string> v; + int i; + for (i = 0; !aiter.end() && i < 3; ++aiter, ++i) { + JSONObj *o = *aiter; + v.push_back(o->get_data()); + } + if (i != 3 || !aiter.end()) { /* we expect exactly 3 arguments here */ + err_msg = "Bad condition array, expecting 3 arguments"; + return -EINVAL; + } + + int r = add_condition(v[0], v[1], v[2], err_msg); + if (r < 0) + return r; + } else { + add_simple_check(child->get_name(), child->get_data()); + } + } + return 0; +} diff --git a/src/rgw/rgw_policy_s3.h b/src/rgw/rgw_policy_s3.h new file mode 100644 index 00000000000..84a2ee71751 --- /dev/null +++ b/src/rgw/rgw_policy_s3.h @@ -0,0 +1,56 @@ +#ifndef CEPH_RGW_POLICY_H +#define CEPH_RGW_POLICY_H + +#include <limits.h> + +#include <map> +#include <list> +#include <string> + +#include "include/utime.h" + +#include "rgw_string.h" + + +class RGWPolicyEnv { + std::map<std::string, std::string, ltstr_nocase> vars; + +public: + void add_var(const string& name, const string& value); + bool get_var(const string& name, string& val); + bool get_value(const string& s, string& val, std::map<std::string, bool, ltstr_nocase>& checked_vars); + bool match_policy_vars(map<string, bool, ltstr_nocase>& policy_vars, string& err_msg); +}; + +class RGWPolicyCondition; + + +class RGWPolicy { + uint64_t expires; + string expiration_str; + std::list<RGWPolicyCondition *> conditions; + std::list<pair<std::string, std::string> > var_checks; + std::map<std::string, bool, ltstr_nocase> checked_vars; + +public: + off_t min_length; + off_t max_length; + + RGWPolicy() : expires(0), min_length(0), max_length(LLONG_MAX) {} + ~RGWPolicy(); + + int set_expires(const string& e); + + void set_var_checked(const std::string& var) { + checked_vars[var] = true; + } + + int add_condition(const std::string& op, const std::string& first, const std::string& second, string& err_msg); + void add_simple_check(const std::string& var, const std::string& value) { + var_checks.push_back(pair<string, string>(var, value)); + } + + int check(RGWPolicyEnv *env, string& err_msg); + int from_json(bufferlist& bl, string& err_msg); +}; +#endif diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index d4f159bce8d..c2ea5253a54 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -163,6 +163,55 @@ void dump_etag(struct req_state *s, const char *etag) } } +void dump_pair(struct req_state *s, const char *key, const char *value) +{ + if ( (strlen(key) > 0) && (strlen(value) > 0)) + s->cio->print("%s: %s\n", key, value); +} + +void dump_bucket_from_state(struct req_state *s) +{ + if (!s->bucket_name_str.empty()) + s->cio->print("Bucket: \"%s\"\n", s->bucket_name_str.c_str()); +} + +void dump_object_from_state(struct req_state *s) +{ + if (!s->object_str.empty()) + s->cio->print("Key: \"%s\"\n", s->object_str.c_str()); +} + +void dump_uri_from_state(struct req_state *s) +{ + if (strcmp(s->request_uri.c_str(), "/") == 0) { + + string location = "http://"; + location += s->env->get("SERVER_NAME"); + if (!location.empty()) { + location += "/"; + if (!s->bucket_name_str.empty()) { + location += s->bucket_name_str; + location += "/"; + if (!s->object_str.empty()) { + location += s->object_str; + s->cio->print("Location: %s\n", location.c_str()); + } + } + } + } + else { + s->cio->print("Location: \"%s\"\n", s->request_uri.c_str()); + } +} + +void dump_redirect(struct req_state *s, const string& redirect) +{ + if (redirect.empty()) + return; + + s->cio->print("Location: %s\n", redirect.c_str()); +} + void dump_last_modified(struct req_state *s, time_t t) { @@ -323,14 +372,9 @@ int RESTArgs::get_uint64(struct req_state *s, const string& name, uint64_t def_v return 0; } - char *end; - - *val = (uint64_t)strtoull(sval.c_str(), &end, 10); - if (*val == ULLONG_MAX) - return -EINVAL; - - if (*end) - return -EINVAL; + int r = stringtoull(sval, val); + if (r < 0) + return r; return 0; } @@ -348,14 +392,9 @@ int RESTArgs::get_int64(struct req_state *s, const string& name, int64_t def_val return 0; } - char *end; - - *val = (int64_t)strtoll(sval.c_str(), &end, 10); - if (*val == LLONG_MAX) - return -EINVAL; - - if (*end) - return -EINVAL; + int r = stringtoll(sval, val); + if (r < 0) + return r; return 0; } @@ -503,6 +542,22 @@ int RGWPutObj_ObjStore::get_data(bufferlist& bl) return len; } +int RGWPostObj_ObjStore::verify_params() +{ + /* check that we have enough memory to store the object + note that this test isn't exact and may fail unintentionally + for large requests is */ + if (!s->length) { + return -ERR_LENGTH_REQUIRED; + } + off_t len = atoll(s->length); + if (len > (off_t)RGW_MAX_PUT_SIZE) { + return -ERR_TOO_LARGE; + } + + return 0; +} + int RGWPutACLs_ObjStore::get_params() { size_t cl = 0; @@ -919,8 +974,10 @@ int RGWHandler_ObjStore::read_permissions(RGWOp *op_obj) break; } /* is it a 'create bucket' request? */ - if (s->object_str.size() == 0) + if ((s->op == OP_PUT) && s->object_str.size() == 0) return 0; + only_bucket = true; + break; case OP_DELETE: only_bucket = true; break; @@ -982,6 +1039,7 @@ RGWRESTMgr::~RGWRESTMgr() int RGWREST::preprocess(struct req_state *s, RGWClientIO *cio) { s->cio = cio; + s->script_uri = s->env->get("SCRIPT_URI"); s->request_uri = s->env->get("REQUEST_URI"); int pos = s->request_uri.find('?'); if (pos >= 0) { diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index 69056cb2a25..4ac5a5383e3 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -106,6 +106,15 @@ public: int get_data(bufferlist& bl); }; +class RGWPostObj_ObjStore : public RGWPostObj +{ +public: + RGWPostObj_ObjStore() {} + ~RGWPostObj_ObjStore() {} + + virtual int verify_params(); +}; + class RGWPutMetadata_ObjStore : public RGWPutMetadata { public: @@ -282,5 +291,13 @@ extern void dump_range(struct req_state *s, uint64_t ofs, uint64_t end, uint64_t extern void dump_continue(struct req_state *s); extern void list_all_buckets_end(struct req_state *s); extern void dump_time(struct req_state *s, const char *name, time_t *t); +extern void dump_bucket_from_state(struct req_state *s); +extern void dump_object_from_state(struct req_state *s); +extern void dump_uri_from_state(struct req_state *s); +extern void dump_redirect(struct req_state *s, const string& redirect); +extern void dump_pair(struct req_state *s, const char *key, const char *value); +extern bool is_valid_url(const char *url); + + #endif diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 59b15bd7e41..f9daa83c93b 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -3,10 +3,12 @@ #include "common/ceph_crypto.h" #include "common/Formatter.h" +#include "common/utf8.h" #include "rgw_rest.h" #include "rgw_rest_s3.h" #include "rgw_acl.h" +#include "rgw_policy_s3.h" #include "common/armor.h" @@ -16,6 +18,34 @@ using namespace ceph::crypto; +void dump_common_s3_headers(struct req_state *s, const char *etag, + size_t content_len, const char *conn_status) +{ + // how many elements do we expect to include in the response + unsigned int expected_var_len = 4; + map<string, string> head_var; + + utime_t date = ceph_clock_now(s->cct); + if (!date.is_zero()) { + char buf[TIME_BUF_SIZE]; + date.sprintf(buf, TIME_BUF_SIZE); + head_var["date"] = buf; + } + + head_var["etag"] = etag; + head_var["conn_stat"] = conn_status; + head_var["server"] = s->env->get("HTTP_HOST"); + + // if we have all the variables we want go ahead and dump + if (head_var.size() == expected_var_len) { + dump_pair(s, "Date", head_var["date"].c_str()); + dump_etag(s, head_var["etag"].c_str()); + dump_content_length(s, content_len); + dump_pair(s, "Connection", head_var["conn_stat"].c_str()); + dump_pair(s, "Server", head_var["server"].c_str()); + } +} + void list_all_buckets_start(struct req_state *s) { s->formatter->open_array_section_in_ns("ListAllMyBucketsResult", @@ -334,6 +364,751 @@ void RGWPutObj_ObjStore_S3::send_response() end_header(s); } +string trim_whitespace(const string& src) +{ + if (src.empty()) { + return string(); + } + + int start = 0; + for (; start != (int)src.size(); start++) { + if (!isspace(src[start])) + break; + } + + int end = src.size() - 1; + if (end <= start) { + return string(); + } + + for (; end > start; end--) { + if (!isspace(src[end])) + break; + } + + return src.substr(start, end - start + 1); +} + +string trim_quotes(const string& val) +{ + string s = trim_whitespace(val); + if (s.size() < 2) + return s; + + int start = 0; + int end = s.size() - 1; + int quotes_count = 0; + + if (s[start] == '"') { + start++; + quotes_count++; + } + if (s[end] == '"') { + end--; + quotes_count++; + } + if (quotes_count == 2) { + return s.substr(start, end - start + 1); + } + return s; +} + +/* + * parses params in the format: 'first; param1=foo; param2=bar' + */ +static void parse_params(const string& params_str, string& first, map<string, string>& params) +{ + int pos = params_str.find(';'); + if (pos < 0) { + first = trim_whitespace(params_str); + return; + } + + first = trim_whitespace(params_str.substr(0, pos)); + + pos++; + + while (pos < (int)params_str.size()) { + ssize_t end = params_str.find(';', pos); + if (end < 0) + end = params_str.size(); + + string param = params_str.substr(pos, end - pos); + + int eqpos = param.find('='); + if (eqpos > 0) { + string param_name = trim_whitespace(param.substr(0, eqpos)); + string val = trim_quotes(param.substr(eqpos + 1)); + params[param_name] = val; + } else { + params[trim_whitespace(param)] = ""; + } + + pos = end + 1; + } +} + +static int parse_part_field(const string& line, string& field_name, struct post_part_field& field) +{ + int pos = line.find(':'); + if (pos < 0) + return -EINVAL; + + field_name = line.substr(0, pos); + if (pos >= (int)line.size() - 1) + return 0; + + parse_params(line.substr(pos + 1), field.val, field.params); + + return 0; +} + +bool is_crlf(const char *s) +{ + return (*s == '\r' && *(s + 1) == '\n'); +} + +/* + * find the index of the boundary, if exists, or optionally the next end of line + * also returns how many bytes to skip + */ +static int index_of(bufferlist& bl, int max_len, const string& str, bool check_crlf, + bool *reached_boundary, int *skip) +{ + *reached_boundary = false; + *skip = 0; + + if (str.size() < 2) // we assume boundary is at least 2 chars (makes it easier with crlf checks) + return -EINVAL; + + if (bl.length() < str.size()) + return -1; + + const char *buf = bl.c_str(); + const char *s = str.c_str(); + + if (max_len > (int)bl.length()) + max_len = bl.length(); + + int i; + for (i = 0; i < max_len; i++, buf++) { + if (check_crlf && + i >= 1 && + is_crlf(buf - 1)) { + return i + 1; // skip the crlf + } + if ((i < max_len - (int)str.size() + 1) && + (buf[0] == s[0] && buf[1] == s[1]) && + (strncmp(buf, s, str.size()) == 0)) { + *reached_boundary = true; + *skip = str.size(); + + /* oh, great, now we need to swallow the preceding crlf + * if exists + */ + if ((i >= 2) && + is_crlf(buf - 2)) { + i -= 2; + *skip += 2; + } + return i; + } + } + + return -1; +} + +int RGWPostObj_ObjStore_S3::read_with_boundary(bufferlist& bl, uint64_t max, bool check_crlf, + bool *reached_boundary, bool *done) +{ + uint64_t cl = max + 2 + boundary.size(); + + if (max > in_data.length()) { + uint64_t need_to_read = cl - in_data.length(); + + bufferptr bp(need_to_read); + + int read_len; + s->cio->read(bp.c_str(), need_to_read, &read_len); + + in_data.append(bp, 0, read_len); + } + + *done = false; + int skip; + int index = index_of(in_data, cl, boundary, check_crlf, reached_boundary, &skip); + if (index >= 0) + max = index; + + if (max > in_data.length()) + max = in_data.length(); + + bl.substr_of(in_data, 0, max); + + bufferlist new_read_data; + + /* + * now we need to skip boundary for next time, also skip any crlf, or + * check to see if it's the last final boundary (marked with "--" at the end + */ + if (*reached_boundary) { + int left = in_data.length() - max; + if (left < skip + 2) { + int need = skip + 2 - left; + bufferptr boundary_bp(need); + int actual; + s->cio->read(boundary_bp.c_str(), need, &actual); + in_data.append(boundary_bp); + } + max += skip; // skip boundary for next time + if (in_data.length() >= max + 2) { + const char *data = in_data.c_str(); + if (is_crlf(data + max)) { + max += 2; + } else { + if (*(data + max) == '-' && + *(data + max + 1) == '-') { + *done = true; + max += 2; + } + } + } + } + + new_read_data.substr_of(in_data, max, in_data.length() - max); + in_data = new_read_data; + + return 0; +} + +int RGWPostObj_ObjStore_S3::read_line(bufferlist& bl, uint64_t max, + bool *reached_boundary, bool *done) +{ + return read_with_boundary(bl, max, true, reached_boundary, done); +} + +int RGWPostObj_ObjStore_S3::read_data(bufferlist& bl, uint64_t max, + bool *reached_boundary, bool *done) +{ + return read_with_boundary(bl, max, false, reached_boundary, done); +} + + +int RGWPostObj_ObjStore_S3::read_form_part_header(struct post_form_part *part, + bool *done) +{ + bufferlist bl; + bool reached_boundary; + int r = read_line(bl, RGW_MAX_CHUNK_SIZE, &reached_boundary, done); + if (r < 0) + return r; + + if (*done) { + return 0; + } + + if (reached_boundary) { // skip the first boundary + r = read_line(bl, RGW_MAX_CHUNK_SIZE, &reached_boundary, done); + if (r < 0) + return r; + if (*done) + return 0; + } + + while (true) { + /* + * iterate through fields + */ + string line = trim_whitespace(string(bl.c_str(), bl.length())); + + if (line.empty()) + break; + + struct post_part_field field; + + string field_name; + r = parse_part_field(line, field_name, field); + if (r < 0) + return r; + + part->fields[field_name] = field; + + if (stringcasecmp(field_name, "Content-Disposition") == 0) { + part->name = field.params["name"]; + } + + if (reached_boundary) + break; + + r = read_line(bl, RGW_MAX_CHUNK_SIZE, &reached_boundary, done); + } + + return 0; +} + +bool RGWPostObj_ObjStore_S3::part_str(const string& name, string *val) +{ + map<string, struct post_form_part, ltstr_nocase>::iterator iter = parts.find(name); + if (iter == parts.end()) + return false; + + bufferlist& data = iter->second.data; + string str = string(data.c_str(), data.length()); + *val = trim_whitespace(str); + return true; +} + +bool RGWPostObj_ObjStore_S3::part_bl(const string& name, bufferlist *pbl) +{ + map<string, struct post_form_part, ltstr_nocase>::iterator iter = parts.find(name); + if (iter == parts.end()) + return false; + + *pbl = iter->second.data; + return true; +} + +void RGWPostObj_ObjStore_S3::rebuild_key(string& key) +{ + static string var = "${filename}"; + int pos = key.find(var); + if (pos < 0) + return; + + string new_key = key.substr(0, pos); + new_key.append(filename); + new_key.append(key.substr(pos + var.size())); + + key = new_key; +} + +int RGWPostObj_ObjStore_S3::get_params() +{ + string temp_line; + string param; + string old_param; + string param_value; + + string whitespaces (" \t\f\v\n\r"); + + // get the part boundary + string req_content_type_str = s->env->get("CONTENT_TYPE"); + string req_content_type; + map<string, string> params; + + if (s->expect_cont) { + /* ok, here it really gets ugly. With POST, the params are embedded in the + * request body, so we need to continue before being able to actually look + * at them. This diverts from the usual request flow. + */ + dump_continue(s); + s->expect_cont = false; + } + + parse_params(req_content_type_str, req_content_type, params); + + if (req_content_type.compare("multipart/form-data") != 0) { + err_msg = "Request Content-Type is not multipart/form-data"; + return -EINVAL; + } + + if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20)) { + ldout(s->cct, 20) << "request content_type_str=" << req_content_type_str << dendl; + ldout(s->cct, 20) << "request content_type params:" << dendl; + map<string, string>::iterator iter; + for (iter = params.begin(); iter != params.end(); ++iter) { + ldout(s->cct, 20) << " " << iter->first << " -> " << iter->second << dendl; + } + } + + ldout(s->cct, 20) << "adding bucket to policy env: " << s->bucket.name << dendl; + env.add_var("bucket", s->bucket.name); + + map<string, string>::iterator iter = params.find("boundary"); + if (iter == params.end()) { + err_msg = "Missing multipart boundary specification"; + return -EINVAL; + } + + // create the boundary + boundary = "--"; + boundary.append(iter->second); + + bool done; + do { + struct post_form_part part; + int r = read_form_part_header(&part, &done); + if (r < 0) + return r; + + if (s->cct->_conf->subsys.should_gather(ceph_subsys_rgw, 20)) { + map<string, struct post_part_field, ltstr_nocase>::iterator piter; + for (piter = part.fields.begin(); piter != part.fields.end(); ++piter) { + ldout(s->cct, 20) << "read part header: name=" << part.name << " content_type=" << part.content_type << dendl; + ldout(s->cct, 20) << "name=" << piter->first << dendl; + ldout(s->cct, 20) << "val=" << piter->second.val << dendl; + ldout(s->cct, 20) << "params:" << dendl; + map<string, string>& params = piter->second.params; + for (iter = params.begin(); iter != params.end(); ++iter) { + ldout(s->cct, 20) << " " << iter->first << " -> " << iter->second << dendl; + } + } + } + + if (done) { /* unexpected here */ + err_msg = "Malformed request"; + return -EINVAL; + } + + if (stringcasecmp(part.name, "file") == 0) { /* beginning of data transfer */ + struct post_part_field& field = part.fields["Content-Disposition"]; + map<string, string>::iterator iter = field.params.find("filename"); + if (iter != field.params.end()) { + filename = iter->second; + } + parts[part.name] = part; + data_pending = true; + break; + } + + bool boundary; + r = read_data(part.data, RGW_MAX_CHUNK_SIZE, &boundary, &done); + if (!boundary) { + err_msg = "Couldn't find boundary"; + return -EINVAL; + } + parts[part.name] = part; + string part_str(part.data.c_str(), part.data.length()); + env.add_var(part.name, part_str); + } while (!done); + + if (!part_str("key", &s->object_str)) { + err_msg = "Key not specified"; + return -EINVAL; + } + + rebuild_key(s->object_str); + + env.add_var("key", s->object_str); + + part_str("Content-Type", &content_type); + env.add_var("Content-Type", content_type); + + map<string, struct post_form_part, ltstr_nocase>::iterator piter = parts.upper_bound(RGW_AMZ_META_PREFIX); + for (; piter != parts.end(); ++piter) { + string n = piter->first; + if (strncasecmp(n.c_str(), RGW_AMZ_META_PREFIX, sizeof(RGW_AMZ_META_PREFIX) - 1) != 0) + break; + + string attr_name = RGW_ATTR_PREFIX; + attr_name.append(n); + + /* need to null terminate it */ + bufferlist& data = piter->second.data; + string str = string(data.c_str(), data.length()); + + bufferlist attr_bl; + attr_bl.append(str.c_str(), str.size() + 1); + + attrs[attr_name] = attr_bl; + } + + int r = get_policy(); + if (r < 0) + return r; + + min_len = post_policy.min_length; + max_len = post_policy.max_length; + + return 0; +} + +int RGWPostObj_ObjStore_S3::get_policy() +{ + bufferlist encoded_policy; + string uid; + + if (part_bl("policy", &encoded_policy)) { + + // check that the signature matches the encoded policy + string s3_access_key; + if (!part_str("AWSAccessKeyId", &s3_access_key)) { + ldout(s->cct, 0) << "No S3 access key found!" << dendl; + err_msg = "Missing access key"; + return -EINVAL; + } + string signature_str; + if (!part_str("signature", &signature_str)) { + ldout(s->cct, 0) << "No signature found!" << dendl; + err_msg = "Missing signature"; + return -EINVAL; + } + + RGWUserInfo user_info; + + ret = rgw_get_user_info_by_access_key(store, s3_access_key, user_info); + if (ret < 0) { + ldout(s->cct, 0) << "User lookup failed!" << dendl; + err_msg = "Bad access key / signature"; + return -EACCES; + } + + map<string, RGWAccessKey> access_keys = user_info.access_keys; + + map<string, RGWAccessKey>::const_iterator iter = access_keys.begin(); + string s3_secret_key = (iter->second).key; + + char calc_signature[CEPH_CRYPTO_HMACSHA1_DIGESTSIZE]; + + calc_hmac_sha1(s3_secret_key.c_str(), s3_secret_key.size(), encoded_policy.c_str(), encoded_policy.length(), calc_signature); + bufferlist encoded_hmac; + bufferlist raw_hmac; + raw_hmac.append(calc_signature, CEPH_CRYPTO_HMACSHA1_DIGESTSIZE); + raw_hmac.encode_base64(encoded_hmac); + encoded_hmac.append((char)0); /* null terminate */ + + if (signature_str.compare(encoded_hmac.c_str()) != 0) { + ldout(s->cct, 0) << "Signature verification failed!" << dendl; + ldout(s->cct, 0) << "expected: " << signature_str.c_str() << dendl; + ldout(s->cct, 0) << "got: " << encoded_hmac.c_str() << dendl; + err_msg = "Bad access key / signature"; + return -EACCES; + } + ldout(s->cct, 0) << "Successful Signature Verification!" << dendl; + bufferlist decoded_policy; + try { + decoded_policy.decode_base64(encoded_policy); + } catch (buffer::error& err) { + ldout(s->cct, 0) << "failed to decode_base64 policy" << dendl; + err_msg = "Could not decode policy"; + return -EINVAL; + } + + decoded_policy.append('\0'); // NULL terminate + + ldout(s->cct, 0) << "POST policy: " << decoded_policy.c_str() << dendl; + + int r = post_policy.from_json(decoded_policy, err_msg); + if (r < 0) { + if (err_msg.empty()) { + err_msg = "Failed to parse policy"; + } + ldout(s->cct, 0) << "failed to parse policy" << dendl; + return -EINVAL; + } + + post_policy.set_var_checked("AWSAccessKeyId"); + post_policy.set_var_checked("policy"); + post_policy.set_var_checked("signature"); + + r = post_policy.check(&env, err_msg); + if (r < 0) { + if (err_msg.empty()) { + err_msg = "Policy check failed"; + } + ldout(s->cct, 0) << "policy check failed" << dendl; + return r; + } + + s->user = user_info; + } else { + ldout(s->cct, 0) << "No attached policy found!" << dendl; + } + + string canned_acl; + part_str("acl", &canned_acl); + + RGWAccessControlPolicy_S3 s3policy(s->cct); + ldout(s->cct, 20) << "canned_acl=" << canned_acl << dendl; + if (!s3policy.create_canned(s->user.user_id, "", canned_acl)) { + err_msg = "Bad canned ACLs"; + return -EINVAL; + } + + policy = s3policy; + + return 0; +} + +int RGWPostObj_ObjStore_S3::complete_get_params() +{ + bool done; + do { + struct post_form_part part; + int r = read_form_part_header(&part, &done); + if (r < 0) + return r; + + bufferlist part_data; + bool boundary; + r = read_data(part.data, RGW_MAX_CHUNK_SIZE, &boundary, &done); + if (!boundary) { + return -EINVAL; + } + + parts[part.name] = part; + } while (!done); + + return 0; +} + +int RGWPostObj_ObjStore_S3::get_data(bufferlist& bl) +{ + bool boundary; + bool done; + + int r = read_data(bl, RGW_MAX_CHUNK_SIZE, &boundary, &done); + if (r < 0) + return r; + + if (boundary) { + data_pending = false; + + if (!done) { /* reached end of data, let's drain the rest of the params */ + r = complete_get_params(); + if (r < 0) + return r; + } + } + + return bl.length(); +} + +static void escape_char(char c, string& dst) +{ + char buf[16]; + snprintf(buf, sizeof(buf), "%%%.2X", (unsigned int)c); + dst.append(buf); +} + +static bool char_needs_url_encoding(char c) +{ + if (c < 0x20 || c >= 0x7f) + return true; + + switch (c) { + case 0x20: + case 0x22: + case 0x23: + case 0x25: + case 0x26: + case 0x2B: + case 0x2C: + case 0x2F: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3E: + case 0x3D: + case 0x3F: + case 0x40: + case 0x5B: + case 0x5D: + case 0x5C: + case 0x5E: + case 0x60: + case 0x7B: + case 0x7D: + return true; + } + return false; +} + +static void url_escape(const string& src, string& dst) +{ + const char *p = src.c_str(); + for (unsigned i = 0; i < src.size(); i++, p++) { + if (char_needs_url_encoding(*p)) { + escape_char(*p, dst); + continue; + } + + dst.append(p, 1); + } +} + +void RGWPostObj_ObjStore_S3::send_response() +{ + if (ret == 0 && parts.count("success_action_redirect")) { + string redirect; + + part_str("success_action_redirect", &redirect); + + string bucket; + string key; + string etag_str = "\""; + + etag_str.append(etag); + etag_str.append("\""); + + string etag_url; + + url_escape(s->bucket_name, bucket); + url_escape(s->object_str, key); + url_escape(etag_str, etag_url); + + + redirect.append("?bucket="); + redirect.append(bucket); + redirect.append("&key="); + redirect.append(key); + redirect.append("&etag="); + redirect.append(etag_url); + + int r = check_utf8(redirect.c_str(), redirect.size()); + if (r < 0) { + ret = r; + goto done; + } + dump_redirect(s, redirect); + ret = STATUS_REDIRECT; + } else if (ret == 0 && parts.count("success_action_status")) { + string status_string; + uint32_t status_int; + + part_str("success_action_status", &status_string); + + int r = stringtoul(status_string, &status_int); + if (r < 0) { + ret = r; + goto done; + } + + switch (status_int) { + case 200: + break; + case 201: + ret = STATUS_CREATED; + break; + default: + ret = STATUS_NO_CONTENT; + break; + } + } else if (!ret) { + ret = STATUS_NO_CONTENT; + } + +done: + if (ret == STATUS_CREATED) { + s->formatter->open_object_section("PostResponse"); + if (g_conf->rgw_dns_name.length()) + s->formatter->dump_format("Location", "%s/%s", s->script_uri.c_str(), s->object_str.c_str()); + s->formatter->dump_string("Bucket", s->bucket_name); + s->formatter->dump_string("Key", s->object_str.c_str()); + s->formatter->close_section(); + } + s->err.message = err_msg; + set_req_state_err(s, ret); + dump_errno(s); + dump_content_length(s, s->formatter->get_len()); + end_header(s); + if (ret != STATUS_CREATED) + return; + + rgw_flush_formatter_and_reset(s, s->formatter); +} + + void RGWDeleteObj_ObjStore_S3::send_response() { int r = ret; @@ -745,7 +1520,7 @@ RGWOp *RGWHandler_ObjStore_Bucket_S3::op_post() return new RGWDeleteMultiObj_ObjStore_S3; } - return NULL; + return new RGWPostObj_ObjStore_S3; } RGWOp *RGWHandler_ObjStore_Obj_S3::get_obj_op(bool get_data) diff --git a/src/rgw/rgw_rest_s3.h b/src/rgw/rgw_rest_s3.h index 5bdec1b0e4f..05ca142f6bc 100644 --- a/src/rgw/rgw_rest_s3.h +++ b/src/rgw/rgw_rest_s3.h @@ -5,6 +5,7 @@ #include "rgw_op.h" #include "rgw_html_errors.h" #include "rgw_acl_s3.h" +#include "rgw_policy_s3.h" #define RGW_AUTH_GRACE_MINS 15 @@ -81,6 +82,53 @@ public: void send_response(); }; +struct post_part_field { + string val; + map<string, string> params; +}; + +struct post_form_part { + string name; + string content_type; + map<string, struct post_part_field, ltstr_nocase> fields; + bufferlist data; +}; + +class RGWPostObj_ObjStore_S3 : public RGWPostObj_ObjStore { + string boundary; + string filename; + bufferlist in_data; + map<string, post_form_part, const ltstr_nocase> parts; + RGWPolicyEnv env; + RGWPolicy post_policy; + string err_msg; + + int read_with_boundary(bufferlist& bl, uint64_t max, bool check_eol, + bool *reached_boundary, + bool *done); + + int read_line(bufferlist& bl, uint64_t max, + bool *reached_boundary, bool *done); + + int read_data(bufferlist& bl, uint64_t max, bool *reached_boundary, bool *done); + + int read_form_part_header(struct post_form_part *part, + bool *done); + bool part_str(const string& name, string *val); + bool part_bl(const string& name, bufferlist *pbl); + + int get_policy(); + void rebuild_key(string& key); +public: + RGWPostObj_ObjStore_S3() {} + ~RGWPostObj_ObjStore_S3() {} + + int get_params(); + int complete_get_params(); + void send_response(); + int get_data(bufferlist& bl); +}; + class RGWDeleteObj_ObjStore_S3 : public RGWDeleteObj_ObjStore { public: RGWDeleteObj_ObjStore_S3() {} diff --git a/src/rgw/rgw_string.h b/src/rgw/rgw_string.h new file mode 100644 index 00000000000..3c881a10a91 --- /dev/null +++ b/src/rgw/rgw_string.h @@ -0,0 +1,94 @@ +#ifndef CEPH_RGW_STRING_H +#define CEPH_RGW_STRING_H + +#include <stdlib.h> +#include <limits.h> + +struct ltstr_nocase +{ + bool operator()(const string& s1, const string& s2) const + { + return strcasecmp(s1.c_str(), s2.c_str()) < 0; + } +}; + +static inline int stringcasecmp(const string& s1, const string& s2) +{ + return strcasecmp(s1.c_str(), s2.c_str()); +} + +static inline int stringcasecmp(const string& s1, const char *s2) +{ + return strcasecmp(s1.c_str(), s2); +} + +static inline int stringcasecmp(const string& s1, int ofs, int size, const string& s2) +{ + return strncasecmp(s1.c_str() + ofs, s2.c_str(), size); +} + +static inline int stringtoll(const string& s, int64_t *val) +{ + char *end; + + long long result = strtoll(s.c_str(), &end, 10); + if (result == LLONG_MAX) + return -EINVAL; + + if (*end) + return -EINVAL; + + *val = (int64_t)result; + + return 0; +} + +static inline int stringtoull(const string& s, uint64_t *val) +{ + char *end; + + unsigned long long result = strtoull(s.c_str(), &end, 10); + if (result == ULLONG_MAX) + return -EINVAL; + + if (*end) + return -EINVAL; + + *val = (uint64_t)result; + + return 0; +} + +static inline int stringtol(const string& s, int32_t *val) +{ + char *end; + + long result = strtol(s.c_str(), &end, 10); + if (result == LONG_MAX) + return -EINVAL; + + if (*end) + return -EINVAL; + + *val = (int32_t)result; + + return 0; +} + +static inline int stringtoul(const string& s, uint32_t *val) +{ + char *end; + + unsigned long result = strtoul(s.c_str(), &end, 10); + if (result == ULONG_MAX) + return -EINVAL; + + if (*end) + return -EINVAL; + + *val = (uint32_t)result; + + return 0; +} + +#endif |