diff options
-rw-r--r-- | GNUmakefile | 4 | ||||
-rw-r--r-- | GNUmakefile.mingw | 4 | ||||
-rw-r--r-- | TODO | 4 | ||||
-rw-r--r-- | inc/libs3.h | 174 | ||||
-rw-r--r-- | inc/request.h | 16 | ||||
-rw-r--r-- | inc/util.h | 2 | ||||
-rw-r--r-- | src/acl.c | 49 | ||||
-rw-r--r-- | src/bucket.c | 40 | ||||
-rw-r--r-- | src/general.c | 48 | ||||
-rw-r--r-- | src/object.c | 52 | ||||
-rw-r--r-- | src/request.c | 178 | ||||
-rw-r--r-- | src/s3.c | 388 | ||||
-rw-r--r-- | src/service.c | 10 | ||||
-rw-r--r-- | src/service_access_logging.c | 556 | ||||
-rw-r--r-- | src/util.c | 4 |
15 files changed, 1338 insertions, 191 deletions
diff --git a/GNUmakefile b/GNUmakefile index 63a097e..79650ef 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -251,8 +251,8 @@ libs3: $(LIBS3_SHARED) $(LIBS3_SHARED_MAJOR) $(BUILD)/lib/libs3.a LIBS3_SOURCES := src/acl.c src/bucket.c src/error_parser.c src/general.c \ src/object.c src/request.c src/request_context.c \ - src/response_headers_handler.c src/service.c \ - src/simplexml.c src/util.c + src/response_headers_handler.c src/service_access_logging.c \ + src/service.c src/simplexml.c src/util.c $(LIBS3_SHARED): $(LIBS3_SOURCES:src/%.c=$(BUILD)/obj/%.do) @mkdir -p $(dir $@) diff --git a/GNUmakefile.mingw b/GNUmakefile.mingw index 3451296..afedb9b 100644 --- a/GNUmakefile.mingw +++ b/GNUmakefile.mingw @@ -144,8 +144,8 @@ libs3: $(LIBS3_SHARED) $(BUILD)/lib/libs3.a LIBS3_SOURCES := src/acl.c src/bucket.c src/error_parser.c src/general.c \ src/object.c src/request.c src/request_context.c \ - src/response_headers_handler.c src/service.c \ - src/simplexml.c src/util.c src/mingw_functions.c + src/response_headers_handler.c src/service_access_logging.c \ + src/service.c src/simplexml.c src/util.c src/mingw_functions.c $(LIBS3_SHARED): $(LIBS3_SOURCES:src/%.c=$(BUILD)/obj/%.o) -@mkdir $(subst /,\,$(dir $@)) @@ -1,7 +1,3 @@ -* Implement service logging support - -* Implement function for generating an HTTP authenticated query string - * Implement functions for generating form stuff for posting to s3 * Write s3 man page diff --git a/inc/libs3.h b/inc/libs3.h index 39d4d7b..c16f9da 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -129,6 +129,12 @@ extern "C" { /** + * S3_MAX_BUCKET_NAME_SIZE is the maximum size of a bucket name. + **/ + +#define S3_MAX_BUCKET_NAME_SIZE 255 + +/** * S3_MAX_KEY_SIZE is the maximum size of keys that Amazon S3 supports. **/ #define S3_MAX_KEY_SIZE 1024 @@ -188,6 +194,17 @@ extern "C" { /** + * This is the maximum number of characters that will be stored in the + * return buffer for the utility function which computes an HTTP authenticated + * query string + **/ +#define S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE \ + (sizeof("https://" S3_HOSTNAME "/") + (S3_MAX_KEY_SIZE * 3) + \ + sizeof("?AWSAccessKeyId=") + 32 + sizeof("&Expires=") + 32 + \ + sizeof("&Signature=") + 28 + 1) + + +/** * This constant is used by the S3_initialize() function, to specify that * the winsock library should be initialized by libs3; only relevent on * Microsoft Windows platforms. @@ -250,15 +267,17 @@ typedef enum S3StatusKeyTooLong , S3StatusUriTooLong , S3StatusXmlParseFailure , - S3StatusBadAclEmailAddressTooLong , - S3StatusBadAclUserIdTooLong , - S3StatusBadAclUserDisplayNameTooLong , - S3StatusBadAclGroupUriTooLong , - S3StatusBadAclPermissionTooLong , - S3StatusTooManyAclGrants , - S3StatusBadAclGrantee , - S3StatusBadAclPermission , - S3StatusAclXmlDocumentTooLarge , + S3StatusEmailAddressTooLong , + S3StatusUserIdTooLong , + S3StatusUserDisplayNameTooLong , + S3StatusGroupUriTooLong , + S3StatusPermissionTooLong , + S3StatusTargetBucketTooLong , + S3StatusTargetPrefixTooLong , + S3StatusTooManyGrants , + S3StatusBadGrantee , + S3StatusBadPermission , + S3StatusXmlDocumentTooLarge , S3StatusNameLookupError , S3StatusFailedToConnect , S3StatusServerFailedVerification , @@ -391,13 +410,16 @@ typedef enum * listing owned buckets * All AWS Users - identifies all authenticated AWS users * All Users - identifies all users + * Log Delivery - identifies the Amazon group responsible for writing + * server access logs into buckets **/ typedef enum { S3GranteeTypeAmazonCustomerByEmail = 0, S3GranteeTypeCanonicalUser = 1, S3GranteeTypeAllAwsUsers = 2, - S3GranteeTypeAllUsers = 3 + S3GranteeTypeAllUsers = 3, + S3GranteeTypeLogDelivery = 4 } S3GranteeType; @@ -1366,6 +1388,40 @@ int64_t S3_get_request_context_timeout(S3RequestContext *requestContext); /** ************************************************************************** + * S3 Utility Functions + ************************************************************************** **/ + +/** + * Generates an HTTP authenticated query string, which may then be used by + * a browser (or other web client) to issue the request. The request is + * implicitly a GET request; Amazon S3 is documented to only support this type + * of authenticated query string request. + * + * @param buffer is the output buffer for the authenticated query string. + * It must be at least S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE bytes in + * length. + * @param bucketContext gives the bucket and associated parameters for the + * request to generate. + * @param key gives the key which the authenticated request will GET. + * @param expires gives the number of seconds since Unix epoch for the + * expiration date of the request; after this time, the request will + * no longer be valid. If this value is negative, the largest + * expiration date possible is used (currently, Jan 19, 2038). + * @param resource gives a sub-resource to be fetched for the request, or NULL + * for none. This should be of the form "?<resource>", i.e. + * "?torrent". + * @return One of: + * S3StatusUriTooLong if, due to an internal error, the generated URI + * is longer than S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE bytes in + * length and thus will not fit into the supplied buffer + * S3StatusOK on success + **/ +S3Status S3_generate_authenticated_query_string + (char *buffer, const S3BucketContext *bucketContext, + const char *key, int64_t expires, const char *resource); + + +/** ************************************************************************** * Service Functions ************************************************************************** **/ @@ -1391,7 +1447,7 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, const S3ListServiceHandler *handler, void *callbackData); - + /** ************************************************************************** * Bucket Functions ************************************************************************** **/ @@ -1719,23 +1775,93 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, const S3ResponseHandler *handler, void *callbackData); -/** - * xxx todo - * Service Logging ... - **/ - +/** ************************************************************************** + * Server Access Log Functions + ************************************************************************** **/ /** - * xxx todo - * function for generating an HTTP authenticated query string + * Gets the service access logging settings for a bucket. The service access + * logging settings specify whether or not the S3 service will write service + * access logs for requests made for the given bucket, and if so, several + * settings controlling how these logs will be written. + * + * @param bucketContext gives the bucket and associated parameters for this + * request; this is the bucket for which service access logging is + * being requested + * @param targetBucketReturn must be passed in as a buffer of at least + * (S3_MAX_BUCKET_NAME_SIZE + 1) bytes in length, and will be filled + * in with the target bucket name for access logging for the given + * bucket, which is the bucket into which access logs for the specified + * bucket will be written. This is returned as an empty string if + * service access logging is not enabled for the given bucket. + * @param targetPrefixReturn must be passed in as a buffer of at least + * (S3_MAX_KEY_SIZE + 1) bytes in length, and will be filled in + * with the key prefix for server access logs for the given bucket, + * or the empty string if no such prefix is specified. + * @param aclGrantCountReturn returns the number of ACL grants that are + * associated with the server access logging for the given bucket. + * @param aclGrants must be passed in as an array of at least + * S3_MAX_ACL_GRANT_COUNT S3AclGrant structures, and these will be + * filled in with the target grants associated with the server access + * logging for the given bucket, whose number is returned in the + * aclGrantCountReturn parameter. These grants will be applied to the + * ACL of any server access logging log files generated by the S3 + * service for the given bucket. + * @param requestContext if non-NULL, gives the S3RequestContext to add this + * request to, and does not perform the request immediately. If NULL, + * performs the request immediately and synchronously. + * @param handler gives the callbacks to call as the request is processed and + * completed + * @param callbackData will be passed in as the callbackData parameter to + * all callbacks for this request **/ - - -/** - * xxx todo - * functions for generating form stuff for posting to s3 +void S3_get_server_access_logging(const S3BucketContext *bucketContext, + char *targetBucketReturn, + char *targetPrefixReturn, + int *aclGrantCountReturn, + S3AclGrant *aclGrants, + S3RequestContext *requestContext, + const S3ResponseHandler *handler, + void *callbackData); + + +/** + * Sets the service access logging settings for a bucket. The service access + * logging settings specify whether or not the S3 service will write service + * access logs for requests made for the given bucket, and if so, several + * settings controlling how these logs will be written. + * + * @param bucketContext gives the bucket and associated parameters for this + * request; this is the bucket for which service access logging is + * being set + * @param targetBucket gives the target bucket name for access logging for the + * given bucket, which is the bucket into which access logs for the + * specified bucket will be written. + * @param targetPrefix is an option parameter which specifies the key prefix + * for server access logs for the given bucket, or NULL if no such + * prefix is to be used. + * @param aclGrantCount specifies the number of ACL grants that are to be + * associated with the server access logging for the given bucket. + * @param aclGrants is as an array of S3AclGrant structures, whose number is + * given by the aclGrantCount parameter. These grants will be applied + * to the ACL of any server access logging log files generated by the + * S3 service for the given bucket. + * @param requestContext if non-NULL, gives the S3RequestContext to add this + * request to, and does not perform the request immediately. If NULL, + * performs the request immediately and synchronously. + * @param handler gives the callbacks to call as the request is processed and + * completed + * @param callbackData will be passed in as the callbackData parameter to + * all callbacks for this request **/ - +void S3_set_server_access_logging(const S3BucketContext *bucketContext, + const char *targetBucket, + const char *targetPrefix, int aclGrantCount, + const S3AclGrant *aclGrants, + S3RequestContext *requestContext, + const S3ResponseHandler *handler, + void *callbackData); + #ifdef __cplusplus } diff --git a/inc/request.h b/inc/request.h index 706001b..afb4929 100644 --- a/inc/request.h +++ b/inc/request.h @@ -51,14 +51,8 @@ typedef struct RequestParams // Request type, affects the HTTP verb used HttpRequestType httpRequestType; - // Protocol to use for request - S3Protocol protocol; - - // URI style to use for request - S3UriStyle uriStyle; - - // Bucket name, if any - const char *bucketName; + // Bucket context for request + S3BucketContext bucketContext; // Key, if any const char *key; @@ -69,12 +63,6 @@ typedef struct RequestParams // sub resource, like ?acl, ?location, ?torrent, ?logging const char *subResource; - // AWS Access Key ID - const char *accessKeyId; - - // AWS Secret Access Key - const char *secretAccessKey; - // If this is a copy operation, this gives the source bucket const char *copySourceBucketName; @@ -73,7 +73,7 @@ uint64_t parseUnsignedInt(const char *str); // base64 encode bytes. The output buffer must have at least // ((4 * (inLen + 1)) / 3) bytes in it. Returns the number of bytes written // to [out]. -int base64Encode(const unsigned char *in, int inLen, unsigned char *out); +int base64Encode(const unsigned char *in, int inLen, char *out); // Compute HMAC-SHA-1 with key [key] and message [message], storing result // in [hmac] @@ -70,7 +70,7 @@ static S3Status getAclDataCallback(int bufferSize, const char *buffer, string_buffer_append(gaData->aclXmlDocument, buffer, bufferSize, fit); - return fit ? S3StatusOK : S3StatusAclXmlDocumentTooLarge; + return fit ? S3StatusOK : S3StatusXmlDocumentTooLarge; } @@ -122,14 +122,14 @@ void S3_get_acl(const S3BucketContext *bucketContext, const char *key, RequestParams params = { HttpRequestTypeGET, // httpRequestType - bucketContext->protocol, // protocol - bucketContext->uriStyle, // uriStyle - bucketContext->bucketName, // bucketName + { bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey }, // secretAccessKey key, // key 0, // queryParams "?acl", // subResource - bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, // secretAccessKey 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions @@ -168,7 +168,7 @@ static S3Status generateAclXmlDocument(const char *ownerId, xmlDocumentBufferSize - *xmlDocumentLenReturn - 1, \ fmt, __VA_ARGS__); \ if (*xmlDocumentLenReturn >= xmlDocumentBufferSize) { \ - return S3StatusAclXmlDocumentTooLarge; \ + return S3StatusXmlDocumentTooLarge; \ } \ } while (0) @@ -191,11 +191,24 @@ static S3Status generateAclXmlDocument(const char *ownerId, grant->grantee.canonicalUser.id, grant->grantee.canonicalUser.displayName); break; - default: // case S3GranteeTypeAllAwsUsers/S3GranteeTypeAllUsers: - append("Group\"><URI>http://acs.amazonaws.com/groups/" - "global/%s</URI>", - (grant->granteeType == S3GranteeTypeAllAwsUsers) ? - "AuthenticatedUsers" : "AllUsers"); + default: { // case S3GranteeTypeAllAwsUsers/S3GranteeTypeAllUsers: + const char *grantee; + switch (grant->granteeType) { + case S3GranteeTypeAllAwsUsers: + grantee = "http://acs.amazonaws.com/groups/global/" + "AuthenticatedUsers"; + break; + case S3GranteeTypeAllUsers: + grantee = "http://acs.amazonaws.com/groups/global/" + "AllUsers"; + break; + default: + grantee = "http://acs.amazonaws.com/groups/s3/" + "LogDelivery"; + break; + } + append("Group\"><URI>%s</URI>", grantee); + } break; } append("</Grantee><Permission>%s</Permission></Grant>", @@ -278,7 +291,7 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, { if (aclGrantCount > S3_MAX_ACL_GRANT_COUNT) { (*(handler->completeCallback)) - (S3StatusTooManyAclGrants, 0, callbackData); + (S3StatusTooManyGrants, 0, callbackData); return; } @@ -309,14 +322,14 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, RequestParams params = { HttpRequestTypePUT, // httpRequestType - bucketContext->protocol, // protocol - bucketContext->uriStyle, // uriStyle - bucketContext->bucketName, // bucketName + { bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey }, // secretAccessKey key, // key 0, // queryParams "?acl", // subResource - bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, // secretAccessKey 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions diff --git a/src/bucket.c b/src/bucket.c index ccedd0b..4f56cb9 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -131,14 +131,14 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, RequestParams params = { HttpRequestTypeGET, // httpRequestType - protocol, // protocol - uriStyle, // uriStyle - bucketName, // bucketName + { bucketName, // bucketName + protocol, // protocol + uriStyle, // uriStyle + accessKeyId, // accessKeyId + secretAccessKey }, // secretAccessKey 0, // key 0, // queryParams "?location", // subResource - accessKeyId, // accessKeyId - secretAccessKey, // secretAccessKey 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions @@ -267,14 +267,14 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, RequestParams params = { HttpRequestTypePUT, // httpRequestType - protocol, // protocol - S3UriStylePath, // uriStyle - bucketName, // bucketName + { bucketName, // bucketName + protocol, // protocol + S3UriStylePath, // uriStyle + accessKeyId, // accessKeyId + secretAccessKey }, // secretAccessKey 0, // key 0, // queryParams 0, // subResource - accessKeyId, // accessKeyId - secretAccessKey, // secretAccessKey 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions @@ -349,14 +349,14 @@ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, RequestParams params = { HttpRequestTypeDELETE, // httpRequestType - protocol, // protocol - uriStyle, // uriStyle - bucketName, // bucketName + { bucketName, // bucketName + protocol, // protocol + uriStyle, // uriStyle + accessKeyId, // accessKeyId + secretAccessKey }, // secretAccessKey 0, // key 0, // queryParams 0, // subResource - accessKeyId, // accessKeyId - secretAccessKey, // secretAccessKey 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions @@ -700,14 +700,14 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, RequestParams params = { HttpRequestTypeGET, // httpRequestType - bucketContext->protocol, // protocol - bucketContext->uriStyle, // uriStyle - bucketContext->bucketName, // bucketName + { bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey }, // secretAccessKey 0, // key queryParams[0] ? queryParams : 0, // queryParams 0, // subResource - bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, // secretAccessKey 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions diff --git a/src/general.c b/src/general.c index a9645f0..861c289 100644 --- a/src/general.c +++ b/src/general.c @@ -90,15 +90,17 @@ const char *S3_get_status_name(S3Status status) handlecase(KeyTooLong); handlecase(UriTooLong); handlecase(XmlParseFailure); - handlecase(BadAclEmailAddressTooLong); - handlecase(BadAclUserIdTooLong); - handlecase(BadAclUserDisplayNameTooLong); - handlecase(BadAclGroupUriTooLong); - handlecase(BadAclPermissionTooLong); - handlecase(TooManyAclGrants); - handlecase(BadAclGrantee); - handlecase(BadAclPermission); - handlecase(AclXmlDocumentTooLarge); + handlecase(EmailAddressTooLong); + handlecase(UserIdTooLong); + handlecase(UserDisplayNameTooLong); + handlecase(GroupUriTooLong); + handlecase(PermissionTooLong); + handlecase(TargetBucketTooLong); + handlecase(TargetPrefixTooLong); + handlecase(TooManyGrants); + handlecase(BadGrantee); + handlecase(BadPermission); + handlecase(XmlDocumentTooLarge); handlecase(NameLookupError); handlecase(FailedToConnect); handlecase(ServerFailedVerification); @@ -283,7 +285,7 @@ static S3Status convertAclXmlCallback(const char *elementPath, S3_MAX_GRANTEE_USER_ID_SIZE - caData->ownerIdLen - 1, "%.*s", dataLen, data); if (caData->ownerIdLen >= S3_MAX_GRANTEE_USER_ID_SIZE) { - return S3StatusBadAclUserIdTooLong; + return S3StatusUserIdTooLong; } } else if (!strcmp(elementPath, "AccessControlPolicy/Owner/" @@ -296,7 +298,7 @@ static S3Status convertAclXmlCallback(const char *elementPath, "%.*s", dataLen, data); if (caData->ownerDisplayNameLen >= S3_MAX_GRANTEE_DISPLAY_NAME_SIZE) { - return S3StatusBadAclUserDisplayNameTooLong; + return S3StatusUserDisplayNameTooLong; } } else if (!strcmp(elementPath, @@ -305,7 +307,7 @@ static S3Status convertAclXmlCallback(const char *elementPath, // AmazonCustomerByEmail string_buffer_append(caData->emailAddress, data, dataLen, fit); if (!fit) { - return S3StatusBadAclEmailAddressTooLong; + return S3StatusEmailAddressTooLong; } } else if (!strcmp(elementPath, @@ -314,7 +316,7 @@ static S3Status convertAclXmlCallback(const char *elementPath, // CanonicalUser string_buffer_append(caData->userId, data, dataLen, fit); if (!fit) { - return S3StatusBadAclUserIdTooLong; + return S3StatusUserIdTooLong; } } else if (!strcmp(elementPath, @@ -323,7 +325,7 @@ static S3Status convertAclXmlCallback(const char *elementPath, // CanonicalUser string_buffer_append(caData->userDisplayName, data, dataLen, fit); if (!fit) { - return S3StatusBadAclUserDisplayNameTooLong; + return S3StatusUserDisplayNameTooLong; } } else if (!strcmp(elementPath, @@ -332,7 +334,7 @@ static S3Status convertAclXmlCallback(const char *elementPath, // Group string_buffer_append(caData->groupUri, data, dataLen, fit); if (!fit) { - return S3StatusBadAclGroupUriTooLong; + return S3StatusGroupUriTooLong; } } else if (!strcmp(elementPath, @@ -341,7 +343,7 @@ static S3Status convertAclXmlCallback(const char *elementPath, // Permission string_buffer_append(caData->permission, data, dataLen, fit); if (!fit) { - return S3StatusBadAclPermissionTooLong; + return S3StatusPermissionTooLong; } } } @@ -351,7 +353,7 @@ static S3Status convertAclXmlCallback(const char *elementPath, // A grant has just been completed; so add the next S3AclGrant // based on the values read if (*(caData->aclGrantCountReturn) == S3_MAX_ACL_GRANT_COUNT) { - return S3StatusTooManyAclGrants; + return S3StatusTooManyGrants; } S3AclGrant *grant = &(caData->aclGrants @@ -379,12 +381,17 @@ static S3Status convertAclXmlCallback(const char *elementPath, "AllUsers")) { grant->granteeType = S3GranteeTypeAllUsers; } + else if (!strcmp(caData->groupUri, + "http://acs.amazonaws.com/groups/s3/" + "LogDelivery")) { + grant->granteeType = S3GranteeTypeLogDelivery; + } else { - return S3StatusBadAclGrantee; + return S3StatusBadGrantee; } } else { - return S3StatusBadAclGrantee; + return S3StatusBadGrantee; } if (!strcmp(caData->permission, "READ")) { @@ -403,7 +410,7 @@ static S3Status convertAclXmlCallback(const char *elementPath, grant->permission = S3PermissionFullControl; } else { - return S3StatusBadAclPermission; + return S3StatusBadPermission; } (*(caData->aclGrantCountReturn))++; @@ -466,4 +473,3 @@ int S3_status_is_retryable(S3Status status) return 0; } } - diff --git a/src/object.c b/src/object.c index 0f33655..4c8fd1c 100644 --- a/src/object.c +++ b/src/object.c @@ -42,14 +42,14 @@ void S3_put_object(const S3BucketContext *bucketContext, const char *key, RequestParams params = { HttpRequestTypePUT, // httpRequestType - bucketContext->protocol, // protocol - bucketContext->uriStyle, // uriStyle - bucketContext->bucketName, // bucketName + { bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey }, // secretAccessKey key, // key 0, // queryParams 0, // subResource - bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, // secretAccessKey 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions @@ -196,15 +196,15 @@ void S3_copy_object(const S3BucketContext *bucketContext, const char *key, RequestParams params = { HttpRequestTypeCOPY, // httpRequestType - bucketContext->protocol, // protocol - bucketContext->uriStyle, // uriStyle - destinationBucket ? destinationBucket : - bucketContext->bucketName, // bucketName + { destinationBucket ? destinationBucket : + bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey }, // secretAccessKey destinationKey ? destinationKey : key, // key 0, // queryParams 0, // subResource - bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, // secretAccessKey bucketContext->bucketName, // copySourceBucketName key, // copySourceKey 0, // getConditions @@ -236,14 +236,14 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, RequestParams params = { HttpRequestTypeGET, // httpRequestType - bucketContext->protocol, // protocol - bucketContext->uriStyle, // uriStyle - bucketContext->bucketName, // bucketName + { bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey }, // secretAccessKey key, // key 0, // queryParams 0, // subResource - bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, // secretAccessKey 0, // copySourceBucketName 0, // copySourceKey getConditions, // getConditions @@ -273,14 +273,14 @@ void S3_head_object(const S3BucketContext *bucketContext, const char *key, RequestParams params = { HttpRequestTypeHEAD, // httpRequestType - bucketContext->protocol, // protocol - bucketContext->uriStyle, // uriStyle - bucketContext->bucketName, // bucketName + { bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey }, // secretAccessKey key, // key 0, // queryParams 0, // subResource - bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, // secretAccessKey 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions @@ -310,14 +310,14 @@ void S3_delete_object(const S3BucketContext *bucketContext, const char *key, RequestParams params = { HttpRequestTypeDELETE, // httpRequestType - bucketContext->protocol, // protocol - bucketContext->uriStyle, // uriStyle - bucketContext->bucketName, // bucketName + { bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey }, // secretAccessKey key, // key 0, // queryParams 0, // subResource - bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, // secretAccessKey 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions diff --git a/src/request.c b/src/request.c index 33eac35..9fcd39f 100644 --- a/src/request.c +++ b/src/request.c @@ -634,29 +634,30 @@ static void canonicalize_amz_headers(RequestComputedValues *values) // Canonicalizes the resource into params->canonicalizedResource -static void canonicalize_resource(const RequestParams *params, - RequestComputedValues *values) +static void canonicalize_resource(const char *bucketName, + const char *subResource, + const char *urlEncodedKey, + char *buffer) { - char *buffer = values->canonicalizedResource; int len = 0; *buffer = 0; #define append(str) len += sprintf(&(buffer[len]), "%s", str) - if (params->bucketName && params->bucketName[0]) { + if (bucketName && bucketName[0]) { buffer[len++] = '/'; - append(params->bucketName); + append(bucketName); } append("/"); - if (values->urlEncodedKey[0]) { - append(values->urlEncodedKey); + if (urlEncodedKey && urlEncodedKey[0]) { + append(urlEncodedKey); } - if (params->subResource && params->subResource[0]) { - append(params->subResource); + if (subResource && subResource[0]) { + append(subResource); } } @@ -684,9 +685,9 @@ static S3Status compose_auth_header(const RequestParams *params, { // We allow for: // 17 bytes for HTTP-Verb + \n - // 129 bytes for MD5 + \n + // 129 bytes for Content-MD5 + \n // 129 bytes for Content-Type + \n - // 1 byte for Data + \n + // 1 byte for empty Date + \n // CanonicalizedAmzHeaders & CanonicalizedResource char signbuf[17 + 129 + 129 + 1 + (sizeof(values->canonicalizedAmzHeaders) - 1) + @@ -718,45 +719,48 @@ static S3Status compose_auth_header(const RequestParams *params, // Generate an HMAC-SHA-1 of the signbuf unsigned char hmac[20]; - HMAC_SHA1(hmac, (unsigned char *) params->secretAccessKey, - strlen(params->secretAccessKey), + HMAC_SHA1(hmac, (unsigned char *) params->bucketContext.secretAccessKey, + strlen(params->bucketContext.secretAccessKey), (unsigned char *) signbuf, len); // Now base-64 encode the results - unsigned char b64[((20 + 1) * 4) / 3]; + char b64[((20 + 1) * 4) / 3]; int b64Len = base64Encode(hmac, 20, b64); snprintf(values->authorizationHeader, sizeof(values->authorizationHeader), - "Authorization: AWS %s:%.*s", params->accessKeyId, b64Len, b64); + "Authorization: AWS %s:%.*s", params->bucketContext.accessKeyId, + b64Len, b64); return S3StatusOK; } // Compose the URI to use for the request given the request parameters -static S3Status compose_uri(Request *request, const RequestParams *params, - const RequestComputedValues *values) +static S3Status compose_uri(char *buffer, int bufferSize, + const S3BucketContext *bucketContext, + const char *urlEncodedKey, + const char *subResource, const char *queryParams) { int len = 0; - -#define uri_append(fmt, ...) \ - do { \ - len += snprintf(&(request->uri[len]), \ - sizeof(request->uri) - len, \ - fmt, __VA_ARGS__); \ - if (len >= (int) sizeof(request->uri)) { \ - return S3StatusUriTooLong; \ - } \ + +#define uri_append(fmt, ...) \ + do { \ + len += snprintf(&(buffer[len]), bufferSize - len, fmt, __VA_ARGS__); \ + if (len >= bufferSize) { \ + return S3StatusUriTooLong; \ + } \ } while (0) - uri_append("http%s://", (params->protocol == S3ProtocolHTTP) ? "" : "s"); + uri_append("http%s://", + (bucketContext->protocol == S3ProtocolHTTP) ? "" : "s"); - if (params->bucketName && params->bucketName[0]) { - if (params->uriStyle == S3UriStyleVirtualHost) { - uri_append("%s.s3.amazonaws.com", params->bucketName); + if (bucketContext->bucketName && + bucketContext->bucketName[0]) { + if (bucketContext->uriStyle == S3UriStyleVirtualHost) { + uri_append("%s.s3.amazonaws.com", bucketContext->bucketName); } else { - uri_append("s3.amazonaws.com/%s", params->bucketName); + uri_append("s3.amazonaws.com/%s", bucketContext->bucketName); } } else { @@ -765,18 +769,17 @@ static S3Status compose_uri(Request *request, const RequestParams *params, uri_append("%s", "/"); - if (params->key && params->key[0]) { - uri_append("%s", values->urlEncodedKey); - } - - if (params->subResource && params->subResource[0]) { - uri_append("%s", params->subResource); + uri_append("%s", urlEncodedKey); + + if (subResource && subResource[0]) { + uri_append("%s", subResource); } - if (params->queryParams) { - uri_append("%s", params->queryParams); + if (queryParams) { + uri_append("%c%s", (subResource && subResource[0]) ? '&' : '?', + queryParams); } - + return S3StatusOK; } @@ -985,7 +988,10 @@ static S3Status request_get(const RequestParams *params, request->headers = 0; // Compute the URL - if ((status = compose_uri(request, params, values)) != S3StatusOK) { + if ((status = compose_uri + (request->uri, sizeof(request->uri), + &(params->bucketContext), values->urlEncodedKey, + params->subResource, params->queryParams)) != S3StatusOK) { curl_easy_cleanup(request->curl); free(request); return status; @@ -1109,9 +1115,10 @@ void request_perform(const RequestParams *params, S3RequestContext *context) RequestComputedValues computed; // Validate the bucket name - if (params->bucketName && + if (params->bucketContext.bucketName && ((status = S3_validate_bucket_name - (params->bucketName, params->uriStyle)) != S3StatusOK)) { + (params->bucketContext.bucketName, + params->bucketContext.uriStyle)) != S3StatusOK)) { return_status(status); } @@ -1135,7 +1142,9 @@ void request_perform(const RequestParams *params, S3RequestContext *context) canonicalize_amz_headers(&computed); // Compute the canonicalized resource - canonicalize_resource(params, &computed); + canonicalize_resource(params->bucketContext.bucketName, + params->subResource, computed.urlEncodedKey, + computed.canonicalizedResource); // Compose Authorization header if ((status = compose_auth_header(params, &computed)) != S3StatusOK) { @@ -1283,3 +1292,84 @@ S3Status request_curl_code_to_status(CURLcode code) return S3StatusInternalError; } } + + +S3Status S3_generate_authenticated_query_string + (char *buffer, const S3BucketContext *bucketContext, + const char *key, int64_t expires, const char *resource) +{ +#define MAX_EXPIRES (((int64_t) 1 << 31) - 1) + // S3 seems to only accept expiration dates up to the number of seconds + // representably by a signed 32-bit integer + if (expires < 0) { + expires = MAX_EXPIRES; + } + else if (expires > MAX_EXPIRES) { + expires = MAX_EXPIRES; + } + + // xxx todo: rework this so that it can be incorporated into shared code + // with request_perform(). It's really unfortunate that this code is not + // shared with request_perform(). + + // URL encode the key + char urlEncodedKey[S3_MAX_KEY_SIZE * 3]; + if (key) { + urlEncode(urlEncodedKey, key, strlen(key)); + } + else { + urlEncodedKey[0] = 0; + } + + // Compute canonicalized resource + char canonicalizedResource[MAX_CANONICALIZED_RESOURCE_SIZE]; + canonicalize_resource(bucketContext->bucketName, resource, urlEncodedKey, + canonicalizedResource); + + // We allow for: + // 17 bytes for HTTP-Verb + \n + // 1 byte for empty Content-MD5 + \n + // 1 byte for empty Content-Type + \n + // 20 bytes for Expires + \n + // 0 bytes for CanonicalizedAmzHeaders + // CanonicalizedResource + char signbuf[17 + 1 + 1 + 1 + 20 + sizeof(canonicalizedResource) + 1]; + int len = 0; + +#define signbuf_append(format, ...) \ + len += snprintf(&(signbuf[len]), sizeof(signbuf) - len, \ + format, __VA_ARGS__) + + signbuf_append("%s\n", "GET"); // HTTP-Verb + signbuf_append("%s\n", ""); // Content-MD5 + signbuf_append("%s\n", ""); // Content-Type + signbuf_append("%llu\n", (uint64_t) expires); + signbuf_append("%s", canonicalizedResource); + + // Generate an HMAC-SHA-1 of the signbuf + unsigned char hmac[20]; + + HMAC_SHA1(hmac, (unsigned char *) bucketContext->secretAccessKey, + strlen(bucketContext->secretAccessKey), + (unsigned char *) signbuf, len); + + // Now base-64 encode the results + char b64[((20 + 1) * 4) / 3]; + int b64Len = base64Encode(hmac, 20, b64); + + // Now urlEncode that + char signature[sizeof(b64) * 3]; + urlEncode(signature, b64, b64Len); + + // Finally, compose the uri, with params: + // ?AWSAccessKeyId=xxx[&Expires=]&Signature=xxx + char queryParams[sizeof("AWSAccessKeyId=") + 20 + + sizeof("&Expires=") + 20 + + sizeof("&Signature=") + sizeof(signature) + 1]; + + sprintf(queryParams, "AWSAccessKeyId=%s&Expires=%ld&Signature=%s", + bucketContext->accessKeyId, (long) expires, signature); + + return compose_uri(buffer, S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE, + bucketContext, urlEncodedKey, resource, queryParams); +} @@ -126,6 +126,12 @@ static char errorDetailsG[4096] = { 0 }; #define ALL_DETAILS_PREFIX_LEN (sizeof(ALL_DETAILS_PREFIX) - 1) #define NO_STATUS_PREFIX "noStatus=" #define NO_STATUS_PREFIX_LEN (sizeof(NO_STATUS_PREFIX) - 1) +#define RESOURCE_PREFIX "resource=" +#define RESOURCE_PREFIX_LEN (sizeof(RESOURCE_PREFIX) - 1) +#define TARGET_BUCKET_PREFIX "targetBucket=" +#define TARGET_BUCKET_PREFIX_LEN (sizeof(TARGET_BUCKET_PREFIX) - 1) +#define TARGET_PREFIX_PREFIX "targetPrefix=" +#define TARGET_PREFIX_PREFIX_LEN (sizeof(TARGET_PREFIX_PREFIX) - 1) // util ---------------------------------------------------------------------- @@ -211,6 +217,17 @@ static void usageExit(FILE *out) " <bucket>[/<key>] : Bucket or bucket/key to set the ACL of\n" " [filename] : Input filename for ACL (default is stdin)\n" "\n" +" getlogging : Get the logging status of a bucket\n" +" <bucket> : Bucket to get the logging status of\n" +" [filename] : Output filename for ACL (default is stdout)\n" +"\n" +" setlogging : Set the logging status of a bucket\n" +" <bucket> : Bucket to set the logging status of\n" +" [targetBucket] : Target bucket to log to; if not present, disables\n" +" logging\n" +" [targetPrefix] : Key prefix to use for logs\n" +" [filename] : Input filename for ACL (default is stdin)\n" +"\n" " put : Puts an object\n" " <bucket>/<key> : Bucket/key to put object to\n" " [filename] : Filename to read source data from " @@ -266,7 +283,13 @@ static void usageExit(FILE *out) " [byteCount] : Number of bytes of byte range to return\n" "\n" " head : Gets only the headers of an object, implies -s\n" -" <buckey>/<key> : Bucket/key of object to get headers of\n" +" <bucket>/<key> : Bucket/key of object to get headers of\n" +"\n" +" gqs : Generates an authenticated query string\n" +" <bucket>[/<key>] : Bucket or bucket/key to generate query string for\n" +" [expires] : Expiration date for query string\n" +" [resource] : Sub-resource of key for query string, including\n" +" leading '?', for example, \"?torrent\"\n" "\n" " Canned ACLs:\n" "\n" @@ -437,7 +460,7 @@ static int checkString(const char *str, const char *format) } -static time_t parseIso8601Time(const char *str) +static int64_t parseIso8601Time(const char *str) { // Check to make sure that it has a valid format if (!checkString(str, "dddd-dd-ddTdd:dd:dd")) { @@ -476,7 +499,7 @@ static time_t parseIso8601Time(const char *str) char *tz = getenv("TZ"); setenv("TZ", "UTC", 1); - time_t ret = mktime(&stm); + int64_t ret = mktime(&stm); if (tz) { setenv("TZ", tz, 1); @@ -604,6 +627,11 @@ static int convert_simple_acl(char *aclXml, char *ownerId, grant->granteeType = S3GranteeTypeAllUsers; aclXml += (sizeof("All Users") - 1); } + else if (!strncmp(aclXml, "Log Delivery", + sizeof("Log Delivery") - 1)) { + grant->granteeType = S3GranteeTypeLogDelivery; + aclXml += (sizeof("Log Delivery") - 1); + } else { return 0; } @@ -1384,7 +1412,7 @@ static void put_object(int argc, char **argv, int optindex) uint64_t contentLength = 0; const char *cacheControl = 0, *contentType = 0, *md5 = 0; const char *contentDispositionFilename = 0, *contentEncoding = 0; - time_t expires = -1; + int64_t expires = -1; S3CannedAcl cannedAcl = S3CannedAclPrivate; int metaPropertiesCount = 0; S3NameValue metaProperties[S3_MAX_METADATA_COUNT]; @@ -1642,7 +1670,7 @@ static void copy_object(int argc, char **argv, int optindex) const char *cacheControl = 0, *contentType = 0; const char *contentDispositionFilename = 0, *contentEncoding = 0; - time_t expires = -1; + int64_t expires = -1; S3CannedAcl cannedAcl = S3CannedAclPrivate; int metaPropertiesCount = 0; S3NameValue metaProperties[S3_MAX_METADATA_COUNT]; @@ -1824,7 +1852,7 @@ static void get_object(int argc, char **argv, int optindex) const char *key = slash; const char *filename = 0; - time_t ifModifiedSince = -1, ifNotModifiedSince = -1; + int64_t ifModifiedSince = -1, ifNotModifiedSince = -1; const char *ifMatch = 0, *ifNotMatch = 0; uint64_t startByte = 0, byteCount = 0; @@ -2014,6 +2042,82 @@ static void head_object(int argc, char **argv, int optindex) } +// generate query string ------------------------------------------------------ + +static void generate_query_string(int argc, char **argv, int optindex) +{ + if (optindex == argc) { + fprintf(stderr, "\nERROR: Missing parameter: bucket[/key]\n"); + usageExit(stderr); + } + + const char *bucketName = argv[optindex]; + const char *key = 0; + + // Split bucket/key + char *slash = argv[optindex++]; + while (*slash && (*slash != '/')) { + slash++; + } + if (*slash) { + *slash++ = 0; + key = slash; + } + else { + key = 0; + } + + int64_t expires = -1; + + const char *resource = 0; + + while (optindex < argc) { + char *param = argv[optindex++]; + if (!strncmp(param, EXPIRES_PREFIX, EXPIRES_PREFIX_LEN)) { + expires = parseIso8601Time(&(param[EXPIRES_PREFIX_LEN])); + if (expires < 0) { + fprintf(stderr, "\nERROR: Invalid expires time " + "value; ISO 8601 time format required\n"); + usageExit(stderr); + } + } + else if (!strncmp(param, RESOURCE_PREFIX, RESOURCE_PREFIX_LEN)) { + resource = &(param[RESOURCE_PREFIX_LEN]); + } + else { + fprintf(stderr, "\nERROR: Unknown param: %s\n", param); + usageExit(stderr); + } + } + + S3_init(); + + S3BucketContext bucketContext = + { + bucketName, + protocolG, + uriStyleG, + accessKeyIdG, + secretAccessKeyG + }; + + char buffer[S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE]; + + S3Status status = S3_generate_authenticated_query_string + (buffer, &bucketContext, key, expires, resource); + + if (status != S3StatusOK) { + printf("Failed to generate authenticated query string: %s\n", + S3_get_status_name(status)); + } + else { + printf("%s\n", buffer); + } + + S3_deinitialize(); +} + + // get acl ------------------------------------------------------------------- void get_acl(int argc, char **argv, int optindex) @@ -2141,10 +2245,14 @@ void get_acl(int argc, char **argv, int optindex) type = "Group"; id = "Authenticated AWS Users"; break; - default: + case S3GranteeTypeAllUsers: type = "Group"; id = "All Users"; break; + default: + type = "Group"; + id = "Log Delivery"; + break; } const char *perm; switch (grant->permission) { @@ -2277,6 +2385,261 @@ void set_acl(int argc, char **argv, int optindex) } +// get logging ---------------------------------------------------------------- + +void get_logging(int argc, char **argv, int optindex) +{ + if (optindex == argc) { + fprintf(stderr, "\nERROR: Missing parameter: bucket\n"); + usageExit(stderr); + } + + const char *bucketName = argv[optindex++]; + const char *filename = 0; + + while (optindex < argc) { + char *param = argv[optindex++]; + if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) { + filename = &(param[FILENAME_PREFIX_LEN]); + } + else { + fprintf(stderr, "\nERROR: Unknown param: %s\n", param); + usageExit(stderr); + } + } + + FILE *outfile = 0; + + if (filename) { + // Stat the file, and if it doesn't exist, open it in w mode + struct stat buf; + if (stat(filename, &buf) == -1) { + outfile = fopen(filename, "w" FOPEN_EXTRA_FLAGS); + } + else { + // Open in r+ so that we don't truncate the file, just in case + // there is an error and we write no bytes, we leave the file + // unmodified + outfile = fopen(filename, "r+" FOPEN_EXTRA_FLAGS); + } + + if (!outfile) { + fprintf(stderr, "\nERROR: Failed to open output file %s: ", + filename); + perror(0); + exit(-1); + } + } + else if (showResponsePropertiesG) { + fprintf(stderr, "\nERROR: getlogging -s requires a filename " + "parameter\n"); + usageExit(stderr); + } + else { + outfile = stdout; + } + + int aclGrantCount; + S3AclGrant aclGrants[S3_MAX_ACL_GRANT_COUNT]; + char targetBucket[S3_MAX_BUCKET_NAME_SIZE]; + char targetPrefix[S3_MAX_KEY_SIZE]; + + S3_init(); + + S3BucketContext bucketContext = + { + bucketName, + protocolG, + uriStyleG, + accessKeyIdG, + secretAccessKeyG + }; + + S3ResponseHandler responseHandler = + { + &responsePropertiesCallback, + &responseCompleteCallback + }; + + do { + S3_get_server_access_logging(&bucketContext, targetBucket, targetPrefix, + &aclGrantCount, aclGrants, 0, + &responseHandler, 0); + } while (S3_status_is_retryable(statusG) && should_retry()); + + if (statusG == S3StatusOK) { + if (targetBucket[0]) { + printf("Target Bucket: %s\n", targetBucket); + if (targetPrefix[0]) { + printf("Target Prefix: %s\n", targetPrefix); + } + fprintf(outfile, "%-6s %-90s %-12s\n", " Type", + " User Identifier", + " Permission"); + fprintf(outfile, "------ " + "---------------------------------------------------------" + "--------------------------------- ------------\n"); + int i; + for (i = 0; i < aclGrantCount; i++) { + S3AclGrant *grant = &(aclGrants[i]); + const char *type; + char composedId[S3_MAX_GRANTEE_USER_ID_SIZE + + S3_MAX_GRANTEE_DISPLAY_NAME_SIZE + 16]; + const char *id; + + switch (grant->granteeType) { + case S3GranteeTypeAmazonCustomerByEmail: + type = "Email"; + id = grant->grantee.amazonCustomerByEmail.emailAddress; + break; + case S3GranteeTypeCanonicalUser: + type = "UserID"; + snprintf(composedId, sizeof(composedId), + "%s (%s)", grant->grantee.canonicalUser.id, + grant->grantee.canonicalUser.displayName); + id = composedId; + break; + case S3GranteeTypeAllAwsUsers: + type = "Group"; + id = "Authenticated AWS Users"; + break; + default: + type = "Group"; + id = "All Users"; + break; + } + const char *perm; + switch (grant->permission) { + case S3PermissionRead: + perm = "READ"; + break; + case S3PermissionWrite: + perm = "WRITE"; + break; + case S3PermissionReadACP: + perm = "READ_ACP"; + break; + case S3PermissionWriteACP: + perm = "WRITE_ACP"; + break; + default: + perm = "FULL_CONTROL"; + break; + } + fprintf(outfile, "%-6s %-90s %-12s\n", type, id, perm); + } + } + else { + printf("Service logging is not enabled for this bucket.\n"); + } + } + else { + printError(); + } + + fclose(outfile); + + S3_deinitialize(); +} + + +// set logging ---------------------------------------------------------------- + +void set_logging(int argc, char **argv, int optindex) +{ + if (optindex == argc) { + fprintf(stderr, "\nERROR: Missing parameter: bucket\n"); + usageExit(stderr); + } + + const char *bucketName = argv[optindex++]; + + const char *targetBucket = 0, *targetPrefix = 0, *filename = 0; + + while (optindex < argc) { + char *param = argv[optindex++]; + if (!strncmp(param, TARGET_BUCKET_PREFIX, TARGET_BUCKET_PREFIX_LEN)) { + targetBucket = &(param[TARGET_BUCKET_PREFIX_LEN]); + } + else if (!strncmp(param, TARGET_PREFIX_PREFIX, + TARGET_PREFIX_PREFIX_LEN)) { + targetPrefix = &(param[TARGET_PREFIX_PREFIX_LEN]); + } + else if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) { + filename = &(param[FILENAME_PREFIX_LEN]); + } + else { + fprintf(stderr, "\nERROR: Unknown param: %s\n", param); + usageExit(stderr); + } + } + + int aclGrantCount = 0; + S3AclGrant aclGrants[S3_MAX_ACL_GRANT_COUNT]; + + if (targetBucket) { + FILE *infile; + + if (filename) { + if (!(infile = fopen(filename, "r" FOPEN_EXTRA_FLAGS))) { + fprintf(stderr, "\nERROR: Failed to open input file %s: ", + filename); + perror(0); + exit(-1); + } + } + else { + infile = stdin; + } + + // Read in the complete ACL + char aclBuf[65536]; + aclBuf[fread(aclBuf, 1, sizeof(aclBuf), infile)] = 0; + char ownerId[S3_MAX_GRANTEE_USER_ID_SIZE]; + char ownerDisplayName[S3_MAX_GRANTEE_DISPLAY_NAME_SIZE]; + + // Parse it + if (!convert_simple_acl(aclBuf, ownerId, ownerDisplayName, + &aclGrantCount, aclGrants)) { + fprintf(stderr, "\nERROR: Failed to parse ACLs\n"); + fclose(infile); + exit(-1); + } + + fclose(infile); + } + + S3_init(); + + S3BucketContext bucketContext = + { + bucketName, + protocolG, + uriStyleG, + accessKeyIdG, + secretAccessKeyG + }; + + S3ResponseHandler responseHandler = + { + &responsePropertiesCallback, + &responseCompleteCallback + }; + + do { + S3_set_server_access_logging(&bucketContext, targetBucket, + targetPrefix, aclGrantCount, aclGrants, + 0, &responseHandler, 0); + } while (S3_status_is_retryable(statusG) && should_retry()); + + if (statusG != S3StatusOK) { + printError(); + } + + S3_deinitialize(); +} + + // main ---------------------------------------------------------------------- int main(int argc, char **argv) @@ -2314,7 +2677,7 @@ int main(int argc, char **argv) break; } default: - fprintf(stderr, "\nERROR: Unknown options: -%c\n", c); + fprintf(stderr, "\nERROR: Unknown option: -%c\n", c); // Usage exit usageExit(stderr); } @@ -2388,12 +2751,21 @@ int main(int argc, char **argv) else if (!strcmp(command, "head")) { head_object(argc, argv, optind); } + else if (!strcmp(command, "gqs")) { + generate_query_string(argc, argv, optind); + } else if (!strcmp(command, "getacl")) { get_acl(argc, argv, optind); } else if (!strcmp(command, "setacl")) { set_acl(argc, argv, optind); } + else if (!strcmp(command, "getlogging")) { + get_logging(argc, argv, optind); + } + else if (!strcmp(command, "setlogging")) { + set_logging(argc, argv, optind); + } else { fprintf(stderr, "Unknown command: %s\n", command); return -1; diff --git a/src/service.c b/src/service.c index edb5347..216b981 100644 --- a/src/service.c +++ b/src/service.c @@ -158,14 +158,14 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, RequestParams params = { HttpRequestTypeGET, // httpRequestType - protocol, // protocol - S3UriStylePath, // uriStyle - 0, // bucketName + { 0, // bucketName + protocol, // protocol + S3UriStylePath, // uriStyle + accessKeyId, // accessKeyId + secretAccessKey }, // secretAccessKey 0, // key 0, // queryParams 0, // subResource - accessKeyId, // accessKeyId - secretAccessKey, // secretAccessKey 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions diff --git a/src/service_access_logging.c b/src/service_access_logging.c new file mode 100644 index 0000000..417809a --- /dev/null +++ b/src/service_access_logging.c @@ -0,0 +1,556 @@ +/** ************************************************************************** + * server_access_logging.c + * + * Copyright 2008 Bryan Ischo <bryan@ischo.com> + * + * This file is part of libs3. + * + * libs3 is free software: you can redistribute it and/or modify it under the + * terms of the GNU General Public License as published by the Free Software + * Foundation, version 3 of the License. + * + * In addition, as a special exception, the copyright holders give + * permission to link the code of this library and its programs with the + * OpenSSL library, and distribute linked combinations including the two. + * + * libs3 is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License version 3 + * along with libs3, in a file named COPYING. If not, see + * <http://www.gnu.org/licenses/>. + * + ************************************************************************** **/ + +#include <stdlib.h> +#include <string.h> +#include "libs3.h" +#include "request.h" + + +// get server access logging--------------------------------------------------- + +typedef struct ConvertBlsData +{ + char *targetBucketReturn; + int targetBucketReturnLen; + char *targetPrefixReturn; + int targetPrefixReturnLen; + int *aclGrantCountReturn; + S3AclGrant *aclGrants; + + string_buffer(emailAddress, S3_MAX_GRANTEE_EMAIL_ADDRESS_SIZE); + string_buffer(userId, S3_MAX_GRANTEE_USER_ID_SIZE); + string_buffer(userDisplayName, S3_MAX_GRANTEE_DISPLAY_NAME_SIZE); + string_buffer(groupUri, 128); + string_buffer(permission, 32); +} ConvertBlsData; + + +static S3Status convertBlsXmlCallback(const char *elementPath, + const char *data, int dataLen, + void *callbackData) +{ + ConvertBlsData *caData = (ConvertBlsData *) callbackData; + + int fit; + + if (data) { + if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/" + "TargetBucket")) { + caData->targetBucketReturnLen += + snprintf(&(caData->targetBucketReturn + [caData->targetBucketReturnLen]), + 255 - caData->targetBucketReturnLen - 1, + "%.*s", dataLen, data); + if (caData->targetBucketReturnLen >= 255) { + return S3StatusTargetBucketTooLong; + } + } + else if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/" + "TargetPrefix")) { + caData->targetPrefixReturnLen += + snprintf(&(caData->targetPrefixReturn + [caData->targetPrefixReturnLen]), + 255 - caData->targetPrefixReturnLen - 1, + "%.*s", dataLen, data); + if (caData->targetPrefixReturnLen >= 255) { + return S3StatusTargetPrefixTooLong; + } + } + else if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/" + "TargetGrants/Grant/Grantee/EmailAddress")) { + // AmazonCustomerByEmail + string_buffer_append(caData->emailAddress, data, dataLen, fit); + if (!fit) { + return S3StatusEmailAddressTooLong; + } + } + else if (!strcmp(elementPath, + "AccessControlPolicy/AccessControlList/Grant/" + "Grantee/ID")) { + // CanonicalUser + string_buffer_append(caData->userId, data, dataLen, fit); + if (!fit) { + return S3StatusUserIdTooLong; + } + } + else if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/" + "TargetGrants/Grant/Grantee/DisplayName")) { + // CanonicalUser + string_buffer_append(caData->userDisplayName, data, dataLen, fit); + if (!fit) { + return S3StatusUserDisplayNameTooLong; + } + } + else if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/" + "TargetGrants/Grant/Grantee/URI")) { + // Group + string_buffer_append(caData->groupUri, data, dataLen, fit); + if (!fit) { + return S3StatusGroupUriTooLong; + } + } + else if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/" + "TargetGrants/Grant/Permission")) { + // Permission + string_buffer_append(caData->permission, data, dataLen, fit); + if (!fit) { + return S3StatusPermissionTooLong; + } + } + } + else { + if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/" + "TargetGrants/Grant")) { + // A grant has just been completed; so add the next S3AclGrant + // based on the values read + if (*(caData->aclGrantCountReturn) == S3_MAX_ACL_GRANT_COUNT) { + return S3StatusTooManyGrants; + } + + S3AclGrant *grant = &(caData->aclGrants + [*(caData->aclGrantCountReturn)]); + + if (caData->emailAddress[0]) { + grant->granteeType = S3GranteeTypeAmazonCustomerByEmail; + strcpy(grant->grantee.amazonCustomerByEmail.emailAddress, + caData->emailAddress); + } + else if (caData->userId[0] && caData->userDisplayName[0]) { + grant->granteeType = S3GranteeTypeCanonicalUser; + strcpy(grant->grantee.canonicalUser.id, caData->userId); + strcpy(grant->grantee.canonicalUser.displayName, + caData->userDisplayName); + } + else if (caData->groupUri[0]) { + if (!strcmp(caData->groupUri, + "http://acs.amazonaws.com/groups/global/" + "AuthenticatedUsers")) { + grant->granteeType = S3GranteeTypeAllAwsUsers; + } + else if (!strcmp(caData->groupUri, + "http://acs.amazonaws.com/groups/global/" + "AllUsers")) { + grant->granteeType = S3GranteeTypeAllUsers; + } + else { + return S3StatusBadGrantee; + } + } + else { + return S3StatusBadGrantee; + } + + if (!strcmp(caData->permission, "READ")) { + grant->permission = S3PermissionRead; + } + else if (!strcmp(caData->permission, "WRITE")) { + grant->permission = S3PermissionWrite; + } + else if (!strcmp(caData->permission, "READ_ACP")) { + grant->permission = S3PermissionReadACP; + } + else if (!strcmp(caData->permission, "WRITE_ACP")) { + grant->permission = S3PermissionWriteACP; + } + else if (!strcmp(caData->permission, "FULL_CONTROL")) { + grant->permission = S3PermissionFullControl; + } + else { + return S3StatusBadPermission; + } + + (*(caData->aclGrantCountReturn))++; + + string_buffer_initialize(caData->emailAddress); + string_buffer_initialize(caData->userId); + string_buffer_initialize(caData->userDisplayName); + string_buffer_initialize(caData->groupUri); + string_buffer_initialize(caData->permission); + } + } + + return S3StatusOK; +} + + +static S3Status convert_bls(char *blsXml, char *targetBucketReturn, + char *targetPrefixReturn, int *aclGrantCountReturn, + S3AclGrant *aclGrants) +{ + ConvertBlsData data; + + data.targetBucketReturn = targetBucketReturn; + data.targetBucketReturn[0] = 0; + data.targetBucketReturnLen = 0; + data.targetPrefixReturn = targetPrefixReturn; + data.targetPrefixReturn[0] = 0; + data.targetPrefixReturnLen = 0; + data.aclGrantCountReturn = aclGrantCountReturn; + data.aclGrants = aclGrants; + *aclGrantCountReturn = 0; + string_buffer_initialize(data.emailAddress); + string_buffer_initialize(data.userId); + string_buffer_initialize(data.userDisplayName); + string_buffer_initialize(data.groupUri); + string_buffer_initialize(data.permission); + + // Use a simplexml parser + SimpleXml simpleXml; + simplexml_initialize(&simpleXml, &convertBlsXmlCallback, &data); + + S3Status status = simplexml_add(&simpleXml, blsXml, strlen(blsXml)); + + simplexml_deinitialize(&simpleXml); + + return status; +} + + +// Use a rather arbitrary max size for the document of 64K +#define BLS_XML_DOC_MAXSIZE (64 * 1024) + + +typedef struct GetBlsData +{ + SimpleXml simpleXml; + + S3ResponsePropertiesCallback *responsePropertiesCallback; + S3ResponseCompleteCallback *responseCompleteCallback; + void *callbackData; + + char *targetBucketReturn; + char *targetPrefixReturn; + int *aclGrantCountReturn; + S3AclGrant *aclGrants; + string_buffer(blsXmlDocument, BLS_XML_DOC_MAXSIZE); +} GetBlsData; + + +static S3Status getBlsPropertiesCallback + (const S3ResponseProperties *responseProperties, void *callbackData) +{ + GetBlsData *gsData = (GetBlsData *) callbackData; + + return (*(gsData->responsePropertiesCallback)) + (responseProperties, gsData->callbackData); +} + + +static S3Status getBlsDataCallback(int bufferSize, const char *buffer, + void *callbackData) +{ + GetBlsData *gsData = (GetBlsData *) callbackData; + + int fit; + + string_buffer_append(gsData->blsXmlDocument, buffer, bufferSize, fit); + + return fit ? S3StatusOK : S3StatusXmlDocumentTooLarge; +} + + +static void getBlsCompleteCallback(S3Status requestStatus, + const S3ErrorDetails *s3ErrorDetails, + void *callbackData) +{ + GetBlsData *gsData = (GetBlsData *) callbackData; + + if (requestStatus == S3StatusOK) { + // Parse the document + requestStatus = convert_bls + (gsData->blsXmlDocument, gsData->targetBucketReturn, + gsData->targetPrefixReturn, gsData->aclGrantCountReturn, + gsData->aclGrants); + } + + (*(gsData->responseCompleteCallback)) + (requestStatus, s3ErrorDetails, gsData->callbackData); + + free(gsData); +} + + +void S3_get_server_access_logging(const S3BucketContext *bucketContext, + char *targetBucketReturn, + char *targetPrefixReturn, + int *aclGrantCountReturn, + S3AclGrant *aclGrants, + S3RequestContext *requestContext, + const S3ResponseHandler *handler, + void *callbackData) +{ + // Create the callback data + GetBlsData *gsData = (GetBlsData *) malloc(sizeof(GetBlsData)); + if (!gsData) { + (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); + return; + } + + gsData->responsePropertiesCallback = handler->propertiesCallback; + gsData->responseCompleteCallback = handler->completeCallback; + gsData->callbackData = callbackData; + + gsData->targetBucketReturn = targetBucketReturn; + gsData->targetPrefixReturn = targetPrefixReturn; + gsData->aclGrantCountReturn = aclGrantCountReturn; + gsData->aclGrants = aclGrants; + string_buffer_initialize(gsData->blsXmlDocument); + *aclGrantCountReturn = 0; + + // Set up the RequestParams + RequestParams params = + { + HttpRequestTypeGET, // httpRequestType + { bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey }, // secretAccessKey + 0, // key + 0, // queryParams + "?logging", // subResource + 0, // copySourceBucketName + 0, // copySourceKey + 0, // getConditions + 0, // startByte + 0, // byteCount + 0, // putProperties + &getBlsPropertiesCallback, // propertiesCallback + 0, // toS3Callback + 0, // toS3CallbackTotalSize + &getBlsDataCallback, // fromS3Callback + &getBlsCompleteCallback, // completeCallback + gsData // callbackData + }; + + // Perform the request + request_perform(¶ms, requestContext); +} + + + +// set server access logging--------------------------------------------------- + +static S3Status generateSalXmlDocument(const char *targetBucket, + const char *targetPrefix, + int aclGrantCount, + const S3AclGrant *aclGrants, + int *xmlDocumentLenReturn, + char *xmlDocument, + int xmlDocumentBufferSize) +{ + *xmlDocumentLenReturn = 0; + +#define append(fmt, ...) \ + do { \ + *xmlDocumentLenReturn += snprintf \ + (&(xmlDocument[*xmlDocumentLenReturn]), \ + xmlDocumentBufferSize - *xmlDocumentLenReturn - 1, \ + fmt, __VA_ARGS__); \ + if (*xmlDocumentLenReturn >= xmlDocumentBufferSize) { \ + return S3StatusXmlDocumentTooLarge; \ + } \ + } while (0) + + append("%s", "<BucketLoggingStatus " + "xmlns=\"http://doc.s3.amazonaws.com/2006-03-01\">"); + + if (targetBucket && targetBucket[0]) { + append("<LoggingEnabled><TargetBucket>%s</TargetBucket>", targetBucket); + append("<TargetPrefix>%s</TargetPrefix>", + targetPrefix ? targetPrefix : ""); + + if (aclGrantCount) { + append("%s", "<TargetGrants>"); + int i; + for (i = 0; i < aclGrantCount; i++) { + append("%s", "<Grant><Grantee " + "xmlns:xsi=\"http://www.w3.org/2001/" + "XMLSchema-instance\" xsi:type=\""); + const S3AclGrant *grant = &(aclGrants[i]); + switch (grant->granteeType) { + case S3GranteeTypeAmazonCustomerByEmail: + append("AmazonCustomerByEmail\"><EmailAddress>%s" + "</EmailAddress>", + grant->grantee.amazonCustomerByEmail.emailAddress); + break; + case S3GranteeTypeCanonicalUser: + append("CanonicalUser\"><ID>%s</ID><DisplayName>%s" + "</DisplayName>", + grant->grantee.canonicalUser.id, + grant->grantee.canonicalUser.displayName); + break; + default: // case S3GranteeTypeAllAwsUsers/S3GranteeTypeAllUsers: + append("Group\"><URI>http://acs.amazonaws.com/groups/" + "global/%s</URI>", + (grant->granteeType == S3GranteeTypeAllAwsUsers) ? + "AuthenticatedUsers" : "AllUsers"); + break; + } + append("</Grantee><Permission>%s</Permission></Grant>", + ((grant->permission == S3PermissionRead) ? "READ" : + (grant->permission == S3PermissionWrite) ? "WRITE" : + (grant->permission == + S3PermissionReadACP) ? "READ_ACP" : + (grant->permission == + S3PermissionWriteACP) ? "WRITE_ACP" : "FULL_CONTROL")); + } + append("%s", "</TargetGrants>"); + } + append("%s", "</LoggingEnabled>"); + } + + append("%s", "</BucketLoggingStatus>"); + + return S3StatusOK; +} + + +typedef struct SetSalData +{ + S3ResponsePropertiesCallback *responsePropertiesCallback; + S3ResponseCompleteCallback *responseCompleteCallback; + void *callbackData; + + int salXmlDocumentLen; + char salXmlDocument[BLS_XML_DOC_MAXSIZE]; + int salXmlDocumentBytesWritten; + +} SetSalData; + + +static S3Status setSalPropertiesCallback + (const S3ResponseProperties *responseProperties, void *callbackData) +{ + SetSalData *paData = (SetSalData *) callbackData; + + return (*(paData->responsePropertiesCallback)) + (responseProperties, paData->callbackData); +} + + +static int setSalDataCallback(int bufferSize, char *buffer, void *callbackData) +{ + SetSalData *paData = (SetSalData *) callbackData; + + int remaining = (paData->salXmlDocumentLen - + paData->salXmlDocumentBytesWritten); + + int toCopy = bufferSize > remaining ? remaining : bufferSize; + + if (!toCopy) { + return 0; + } + + memcpy(buffer, &(paData->salXmlDocument + [paData->salXmlDocumentBytesWritten]), toCopy); + + paData->salXmlDocumentBytesWritten += toCopy; + + return toCopy; +} + + +static void setSalCompleteCallback(S3Status requestStatus, + const S3ErrorDetails *s3ErrorDetails, + void *callbackData) +{ + SetSalData *paData = (SetSalData *) callbackData; + + (*(paData->responseCompleteCallback)) + (requestStatus, s3ErrorDetails, paData->callbackData); + + free(paData); +} + + +void S3_set_server_access_logging(const S3BucketContext *bucketContext, + const char *targetBucket, + const char *targetPrefix, int aclGrantCount, + const S3AclGrant *aclGrants, + S3RequestContext *requestContext, + const S3ResponseHandler *handler, + void *callbackData) +{ + if (aclGrantCount > S3_MAX_ACL_GRANT_COUNT) { + (*(handler->completeCallback)) + (S3StatusTooManyGrants, 0, callbackData); + return; + } + + SetSalData *data = (SetSalData *) malloc(sizeof(SetSalData)); + if (!data) { + (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); + return; + } + + // Convert aclGrants to XML document + S3Status status = generateSalXmlDocument + (targetBucket, targetPrefix, aclGrantCount, aclGrants, + &(data->salXmlDocumentLen), data->salXmlDocument, + sizeof(data->salXmlDocument)); + if (status != S3StatusOK) { + free(data); + (*(handler->completeCallback))(status, 0, callbackData); + return; + } + + data->responsePropertiesCallback = handler->propertiesCallback; + data->responseCompleteCallback = handler->completeCallback; + data->callbackData = callbackData; + + data->salXmlDocumentBytesWritten = 0; + + // Set up the RequestParams + RequestParams params = + { + HttpRequestTypePUT, // httpRequestType + { bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey }, // secretAccessKey + 0, // key + 0, // queryParams + "?logging", // subResource + 0, // copySourceBucketName + 0, // copySourceKey + 0, // getConditions + 0, // startByte + 0, // byteCount + 0, // putProperties + &setSalPropertiesCallback, // propertiesCallback + &setSalDataCallback, // toS3Callback + data->salXmlDocumentLen, // toS3CallbackTotalSize + 0, // fromS3Callback + &setSalCompleteCallback, // completeCallback + data // callbackData + }; + + // Perform the request + request_perform(¶ms, requestContext); +} @@ -173,12 +173,12 @@ uint64_t parseUnsignedInt(const char *str) } -int base64Encode(const unsigned char *in, int inLen, unsigned char *out) +int base64Encode(const unsigned char *in, int inLen, char *out) { static const char *ENC = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - unsigned char *original_out = out; + char *original_out = out; while (inLen) { // first 6 bits of char 1 |